Skip to content

Commit 4b1775f

Browse files
calccryptocalccrypto
committed
unify external database list processing
1 parent c44b426 commit 4b1775f

File tree

18 files changed

+132
-172
lines changed

18 files changed

+132
-172
lines changed

‎include/dbutils.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,6 @@ OF SUCH DAMAGE.
7373
#include "SinglyLinkedList.h"
7474
#include "debug.h"
7575
#include "template_db.h"
76-
#include "utils.h"
7776
#include "xattrs.h"
7877

7978
#ifdef __cplusplus

‎include/external.h‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -113,18 +113,21 @@ extern const refstr_t EXTERNAL_TYPE_XATTR; /* convenience struct *
113113
#define EXTERNAL_TYPE_USER_DB_LEN (sizeof(EXTERNAL_TYPE_USER_DB_NAME) - 1)
114114
extern const refstr_t EXTERNAL_TYPE_USER_DB; /* convenience struct */
115115

116-
#define EXTERNAL_ATTACH_PREFIX "extdb"
116+
#define EXTERNAL_ATTACH_PREFIX "extdb"
117117

118118
int create_external_tables(const char *name, sqlite3 *db, void *args);
119119

120-
size_t external_create_query(char *sql, const size_t sql_size,
121-
const char *cols, const size_t cols_len,
122-
const char *table, const size_t table_len,
123-
const refstr_t *type,
124-
const refstr_t *extra);
125-
126120
int external_insert(sqlite3 *db, const char *type, const long long int pinode, const char *filename);
127121

122+
typedef int (*external_process_db_f)(struct input *in, void *args,
123+
const long long int pinode,
124+
const char *filename);
125+
126+
size_t external_read_file(struct input *in,
127+
struct work *child,
128+
external_process_db_f func,
129+
void *args);
130+
128131
/*
129132
* external_concatenate
130133
*

‎include/utils.h‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ OF SUCH DAMAGE.
7272
#include "QueuePerThreadPool.h"
7373
#include "bf.h"
7474
#include "config.h"
75+
#include "external.h"
7576
#include "trie.h"
7677

7778
#ifdef __cplusplus
@@ -106,10 +107,6 @@ int shortpath(const char *name, char *nameout, char *endname);
106107
/* descend */
107108
typedef int (*process_nondir_f)(struct work *nondir, struct entry_data *ed, void *nondir_args);
108109

109-
/* returns number of external databases tracked */
110-
typedef size_t (*process_external_db_f)(struct input *in,
111-
struct work *child, void *args);
112-
113110
struct descend_counters {
114111
size_t dirs;
115112
size_t dirs_insitu;
@@ -126,7 +123,7 @@ int descend(QPTPool_t *ctx, const size_t id, void *args,
126123
struct input *in, struct work *work, ino_t inode,
127124
DIR *dir, trie_t *skip, const int skip_db, const int stat_entries,
128125
QPTPoolFunc_t processdir, process_nondir_f processnondir, void *nondir_args,
129-
process_external_db_f process_external_db, void *external_db_args,
126+
external_process_db_f process_external_db, void *external_db_args,
130127
struct descend_counters *counters);
131128

132129
/* ******************************************************* */

‎src/bffuse.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ OF SUCH DAMAGE.
8686

8787
#include <bf.h>
8888
#include <dbutils.h>
89+
#include <utils.h>
8990

9091
char globalmnt[MAXPATH];
9192
int globalmntlen;

‎src/bfresultfuse.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ OF SUCH DAMAGE.
8585

8686
#include <bf.h>
8787
#include <dbutils.h>
88+
#include <utils.h>
8889

8990
#define MAXPTHREAD 1000
9091

‎src/dbutils.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ OF SUCH DAMAGE.
7676
#include "dbutils.h"
7777
#include "external.h"
7878
#include "histogram.h"
79+
#include "utils.h"
7980

8081
const char READDIRPLUS_CREATE[] =
8182
DROP_TABLE(READDIRPLUS)

‎src/external.c‎

Lines changed: 92 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,10 @@ OF SUCH DAMAGE.
6262

6363

6464

65+
#include <errno.h>
66+
#include <fcntl.h>
6567
#include <string.h>
68+
#include <unistd.h>
6669

6770
#include "external.h"
6871
#include "utils.h"
@@ -108,11 +111,11 @@ int create_external_tables(const char *name, sqlite3 *db, void *args) {
108111
}
109112

110113
/* SELECT <cols> FROM <table> [WHERE (type == '<type>') AND [extra]]; */
111-
size_t external_create_query(char *sql, const size_t sql_size,
112-
const char *cols, const size_t cols_len,
113-
const char *table, const size_t table_len,
114-
const refstr_t *type,
115-
const refstr_t *extra) {
114+
static size_t external_create_query(char *sql, const size_t sql_size,
115+
const char *cols, const size_t cols_len,
116+
const char *table, const size_t table_len,
117+
const refstr_t *type,
118+
const refstr_t *extra) {
116119
size_t len = SNFORMAT_S(sql, sql_size, 4,
117120
"SELECT ", (size_t) 7,
118121
cols, cols_len,
@@ -166,6 +169,90 @@ int external_insert(sqlite3 *db, const char *type, const long long int pinode, c
166169
return 0;
167170
}
168171

172+
/* returns 1 for good/not checked, 0 for error */
173+
static int check_is_db(const int check_extdb_valid, const char *path) {
174+
int rc = !check_extdb_valid;
175+
176+
if (check_extdb_valid) {
177+
/* open the path to make sure it eventually resolves to a file */
178+
sqlite3 *extdb = opendb(path, SQLITE_OPEN_READONLY, 0, 0, NULL, NULL);
179+
if (extdb) {
180+
/* make sure this file is a sqlite3 db */
181+
/* can probably skip this check */
182+
char *err = NULL;
183+
if (sqlite3_exec(extdb, "SELECT '' FROM sqlite_master;", NULL, NULL, &err) == SQLITE_OK) {
184+
rc = 1;
185+
}
186+
else {
187+
fprintf(stderr, "Warning: %s is not a db: %s\n",
188+
path, err);
189+
sqlite3_free(err);
190+
}
191+
192+
closedb(extdb);
193+
}
194+
}
195+
196+
return rc;
197+
}
198+
199+
size_t external_read_file(struct input *in,
200+
struct work *child,
201+
external_process_db_f func,
202+
void *args) {
203+
int extdb_list = open(child->name, O_RDONLY);
204+
if (extdb_list < 0) {
205+
const int err = errno;
206+
fprintf(stderr, "Error: Could not open user external database list in %s: %s (%d)\n",
207+
child->name, strerror(err), err);
208+
return 0;
209+
}
210+
211+
size_t rc = 0;
212+
213+
char *line = NULL;
214+
size_t len = 0;
215+
off_t offset = 0;
216+
while (getline_fd(&line, &len, extdb_list, &offset, 512) > 0) {
217+
char extdb_path_stack[MAXPATH];
218+
char *extdb_path = line;
219+
220+
/* resolve relative paths */
221+
if (line[0] != '/') {
222+
char path[MAXPATH];
223+
SNFORMAT_S(path, sizeof(path), 2,
224+
child->name, child->name_len - child->basename_len,
225+
/* basename does not include slash, so don't need to add another one */
226+
line, len);
227+
228+
if (!realpath(path, extdb_path_stack)) {
229+
const int err = errno;
230+
fprintf(stderr, "Error: Could not resolve external database path %s: %s (%d)\n",
231+
path, strerror(err), err);
232+
free(line);
233+
line = NULL;
234+
continue;
235+
}
236+
237+
extdb_path = extdb_path_stack;
238+
}
239+
240+
if (check_is_db(in->check_extdb_valid, extdb_path) == 1){
241+
rc += !func(in, args, child->pinode, extdb_path);
242+
}
243+
244+
free(line);
245+
line = NULL;
246+
}
247+
248+
free(line);
249+
line = NULL;
250+
251+
close(extdb_list);
252+
253+
return rc;
254+
}
255+
169256
int external_concatenate(sqlite3 *db,
170257
const refstr_t *type,
171258
const refstr_t *extra,

‎src/gufi_dir2index.c‎

Lines changed: 7 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ OF SUCH DAMAGE.
6363

6464

6565
#include <errno.h>
66-
#include <fcntl.h>
6766
#include <dirent.h>
6867
#include <inttypes.h>
6968
#include <stdlib.h>
@@ -154,80 +153,14 @@ static int process_nondir(struct work *entry, struct entry_data *ed, void *args)
154153
return 0;
155154
}
156155

157-
static size_t track_external(struct input *in,
158-
struct work *child,
159-
void *args) {
160-
sqlite3 *db = (sqlite3 *) args;
161-
162-
int extdb_list = open(child->name, O_RDONLY);
163-
if (extdb_list < 0) {
164-
const int err = errno;
165-
fprintf(stderr, "Error: Could not open user external database list in %s: %s (%d)\n",
166-
child->name, strerror(err), err);
167-
return 0;
168-
}
169-
170-
size_t rc = 0;
171-
172-
char *line = NULL;
173-
size_t len = 0;
174-
off_t offset = 0;
175-
while (getline_fd(&line, &len, extdb_list, &offset, 512) > 0) {
176-
char extdb_path_stack[MAXPATH];
177-
char *extdb_path = line;
178-
179-
/* resolve relative paths */
180-
if (line[0] != '/') {
181-
char path[MAXPATH];
182-
SNFORMAT_S(path, sizeof(path), 2,
183-
child->name, child->name_len - child->basename_len,
184-
/* basename does not include slash, so don't need to add another one */
185-
line, len);
186-
187-
if (!realpath(path, extdb_path_stack)) {
188-
const int err = errno;
189-
fprintf(stderr, "Error: Could not resolve external database path %s: %s (%d)\n",
190-
path, strerror(err), err);
191-
free(line);
192-
line = NULL;
193-
continue;
194-
}
195-
196-
extdb_path = extdb_path_stack;
197-
}
156+
static int process_external(struct input *in, void *args,
157+
const long long int pinode,
158+
const char *filename) {
159+
(void) in;
198160

199-
if (in->check_extdb_valid) {
200-
/* open the path to make sure it eventually resolves to a file */
201-
sqlite3 *extdb = opendb(extdb_path, SQLITE_OPEN_READONLY, 0, 0, NULL, NULL);
202-
char *err = NULL;
203-
204-
/* make sure this file is a sqlite3 db */
205-
/* can probably skip this check */
206-
if (sqlite3_exec(extdb, "SELECT '' FROM sqlite_master;", NULL, NULL, &err) == SQLITE_OK) {
207-
rc += !external_insert(db, EXTERNAL_TYPE_USER_DB.data, child->pinode, extdb_path);
208-
}
209-
else {
210-
fprintf(stderr, "Warning: Not tracking requested external db: %s: %s\n",
211-
extdb_path, err);
212-
sqlite3_free(err);
213-
}
214-
215-
closedb(extdb);
216-
}
217-
else {
218-
rc += !external_insert(db, EXTERNAL_TYPE_USER_DB.data, child->pinode, extdb_path);
219-
}
220-
221-
free(line);
222-
line = NULL;
223-
}
224-
225-
free(line);
226-
line = NULL;
227-
228-
close(extdb_list);
161+
sqlite3 *db = (sqlite3 *) args;
229162

230-
return rc;
163+
return external_insert(db, EXTERNAL_TYPE_USER_DB_NAME, pinode, filename);
231164
}
232165

233166
static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
@@ -318,7 +251,7 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
318251
descend(ctx, id, pa, in, nda.work, nda.ed.statuso.st_ino, dir,
319252
in->skip, 0, 0,
320253
processdir, process_nondir, &nda,
321-
track_external, nda.db,
254+
process_external, nda.db,
322255
&ctrs);
323256
stopdb(nda.db);
324257

0 commit comments

Comments
 (0)