Skip to content

Commit 77b1810

Browse files
calccryptocalccrypto
committed
per-thread kv table
alternate setstr (still in code) use ptkv_set and ptkv_get can get and set in the same query can also directly get values using SELECT value from PER_THREAD_KV WHERE key == 'key';
1 parent 1ce739b commit 77b1810

File tree

5 files changed

+190
-2
lines changed

5 files changed

+190
-2
lines changed

‎include/gufi_query/per_thread_kv.h

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#ifndef GUFI_QUERY_PER_THREAD_KV_H
2+
#define GUFI_QUERY_PER_THREAD_KV_H
3+
4+
#include "dbutils.h"
5+
6+
#define PER_THREAD_KV "temp.PER_THREAD_KV"
7+
#define KEY "key"
8+
#define VALUE "value"
9+
#define PER_THREAD_KV_CREATE \
10+
"DROP TABLE IF EXISTS " PER_THREAD_KV ";" \
11+
"CREATE TABLE " PER_THREAD_KV "(" KEY " TEXT, " VALUE " TEXT);"
12+
13+
void ptkv_set(sqlite3_context *context, int argc, sqlite3_value **argv);
14+
int ptkv_set_internal(const size_t tid, sqlite3 *db,
15+
const char *key, const size_t key_len,
16+
const char *val, const size_t val_len);
17+
void ptkv_get(sqlite3_context *context, int argc, sqlite3_value **argv);
18+
19+
#endif

‎src/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ add_library(gufi_query_lib OBJECT
185185
gufi_query/external.c
186186
gufi_query/gqw.c
187187
gufi_query/handle_sql.c
188+
gufi_query/per_thread_kv.c
188189
gufi_query/process_queries.c
189190
gufi_query/processdir.c
190191
gufi_query/query.c

‎src/gufi_query/PoolArgs.c

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ OF SUCH DAMAGE.
7272

7373
#include "gufi_query/PoolArgs.h"
7474
#include "gufi_query/external.h"
75+
#include "gufi_query/per_thread_kv.h"
7576

7677
/* don't introduce undefined behavior by casting function when passing into trie_insert */
7778
static void str_free_wrapper(void *str) {
@@ -159,7 +160,7 @@ int PoolArgs_init(PoolArgs_t *pa, struct input *in, pthread_mutex_t *global_mute
159160
ta->user_strs = trie_alloc();
160161

161162
/*
162-
* {s} defaults to the source tree path - can change, but generally do not
163+
* {s} defaults to the source tree path - can change, but generally will not
163164
*
164165
* maybe move this into processdir?
165166
*/
@@ -181,6 +182,31 @@ int PoolArgs_init(PoolArgs_t *pa, struct input *in, pthread_mutex_t *global_mute
181182

182183
char *err = NULL;
183184

185+
/* create the per-thread kv store */
186+
if (sqlite3_exec(ta->outdb, PER_THREAD_KV_CREATE, NULL, NULL, &err) != SQLITE_OK) {
187+
sqlite_print_err_and_free(err, stderr, "Error: Could not create per-thread kv table: %s\n", err);
188+
break;
189+
}
190+
191+
/* add per-thread kv store setter */
192+
if (sqlite3_create_function(ta->outdb, "ptkv_set", 2, SQLITE_UTF8,
193+
ta->outdb, &ptkv_set, NULL, NULL) != SQLITE_OK) {
194+
sqlite_print_err_and_free(NULL, stderr, "Error: Could not create pktv_set function\n");
195+
break;
196+
}
197+
198+
/* add per-thread kv store getter */
199+
if (sqlite3_create_function(ta->outdb, "ptkv_get", 1, SQLITE_UTF8,
200+
ta->outdb, &ptkv_get, NULL, NULL) != SQLITE_OK) {
201+
sqlite_print_err_and_free(NULL, stderr, "Error: Could not create pktv_get function\n");
202+
break;
203+
}
204+
205+
/* set key s with the source tree path - can change, but generally will not */
206+
if (in->sql_format.source_prefix.data && in->sql_format.source_prefix.len) {
207+
ptkv_set_internal(i, ta->outdb, "s", 1, in->sql_format.source_prefix.data, in->sql_format.source_prefix.len); /* ignore errors? */
208+
}
209+
184210
if (sqlite3_runvt_init(ta->outdb, &err, NULL) != SQLITE_OK) {
185211
sqlite_print_err_and_free(err, stderr, "Error: Could not initialize runvt: %s\n", err);
186212
break;
@@ -277,6 +303,12 @@ void PoolArgs_fin(PoolArgs_t *pa, const size_t allocated) {
277303
for(size_t i = 0; i < allocated; i++) {
278304
ThreadArgs_t *ta = &pa->ta[i];
279305

306+
/* dropping PER_THREAD_KV is not strictly necessary since it is in the temp namespace */
307+
char *err = NULL;
308+
if (sqlite3_exec(ta->outdb, "DROP TABLE IF EXISTS " PER_THREAD_KV ";", NULL, NULL, &err) != SQLITE_OK) {
309+
sqlite_print_err_and_free(err, stderr, "Error: Could not drop per-thread kv table on %s: %s\n", ta->dbname, err);
310+
}
311+
280312
closedb(ta->outdb);
281313

282314
trie_free(ta->user_strs);

‎src/gufi_query/per_thread_kv.c

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#include <stdlib.h>
2+
#include <string.h>
3+
4+
#include "gufi_query/per_thread_kv.h"
5+
#include "str.h"
6+
#include "utils.h"
7+
8+
void ptkv_set(sqlite3_context *context, int argc, sqlite3_value **argv) {
9+
(void) argc;
10+
11+
sqlite3 *db = (sqlite3 *) sqlite3_user_data(context);
12+
const char *key = (const char *) sqlite3_value_text(argv[0]);
13+
const size_t key_len = strlen(key);
14+
const char *val = (const char *) sqlite3_value_text(argv[1]);
15+
const size_t val_len = strlen(val);
16+
17+
#define DELETE "DELETE FROM " PER_THREAD_KV " WHERE " KEY " == "
18+
#define INSERT "INSERT INTO " PER_THREAD_KV "(" KEY ", " VALUE ") VALUES ("
19+
20+
const size_t sql_len = (
21+
((sizeof(DELETE) - 1) +
22+
1 + key_len + 1 + /* 'key' */
23+
1 + 1) + /* ; + space */
24+
((sizeof(INSERT) - 1) +
25+
1 + key_len + 1 + /* 'key' */
26+
1 + 1 + /* , + space */
27+
1 + val_len + 1 + /* 'value' */
28+
1 + 1) /* ); */
29+
);
30+
31+
char *sql = malloc(sql_len + 1);
32+
SNFORMAT_S(sql, sql_len + 1, 10,
33+
DELETE, sizeof(DELETE) - 1,
34+
"'", (size_t) 1,
35+
key, key_len,
36+
"'; ", (size_t) 3,
37+
INSERT, sizeof(INSERT) - 1,
38+
"'", (size_t) 1,
39+
key, key_len,
40+
"', '", (size_t) 4,
41+
val, val_len,
42+
"');", (size_t) 3);
43+
44+
char *err = NULL;
45+
const int rc = sqlite3_exec(db, sql, NULL, NULL, &err);
46+
free(sql);
47+
48+
if (rc != SQLITE_OK) {
49+
sqlite3_result_error(context, err, -1);
50+
sqlite3_free(err);
51+
}
52+
}
53+
54+
int ptkv_set_internal(const size_t tid, sqlite3 *db,
55+
const char *key, const size_t key_len,
56+
const char *val, const size_t val_len) {
57+
#define PTKV_SET "SELECT ptkv_set("
58+
const size_t sql_len = (
59+
(sizeof(PTKV_SET) - 1) +
60+
1 + key_len + 1 + /* 'key' */
61+
1 + 1 + /* , + space */
62+
1 + val_len + 1 + /* 'value' */
63+
1 + 1 /* ); */
64+
);
65+
66+
char *sql = malloc(sql_len + 1);
67+
SNFORMAT_S(sql, sql_len + 1, 6,
68+
PTKV_SET, (sizeof(PTKV_SET) - 1),
69+
"'", (size_t) 1,
70+
key, key_len,
71+
"', '", (size_t) 4,
72+
val, val_len,
73+
"');", (size_t) 3);
74+
75+
char *err = NULL;
76+
const int rc = sqlite3_exec(db, sql, NULL, NULL, &err);
77+
free(sql);
78+
if (rc != SQLITE_OK) {
79+
sqlite_print_err_and_free(err, stderr, "Error: Could not set predefined key \"%s\" on thread %zu: %s\n",
80+
key, tid, err);
81+
return 1;
82+
}
83+
84+
return 0;
85+
}
86+
87+
static int ptkv_get_val(void *args, int count, char **data, char **columns) {
88+
(void) count; (void) columns;
89+
90+
str_t *str = (str_t *) args;
91+
str->len = strlen(data[0]);
92+
str->data = malloc(str->len + 1);
93+
memcpy(str->data, data[0], str->len);
94+
str->data[str->len] = '\0';
95+
return 0;
96+
}
97+
98+
void ptkv_get(sqlite3_context *context, int argc, sqlite3_value **argv) {
99+
(void) argc;
100+
101+
sqlite3 *db = (sqlite3 *) sqlite3_user_data(context);
102+
const char *key = (const char *) sqlite3_value_text(argv[0]);
103+
const size_t key_len = strlen(key);
104+
105+
#define SELECT "SELECT " VALUE " FROM " PER_THREAD_KV " WHERE " KEY " == "
106+
107+
const size_t sql_len = (
108+
(sizeof(SELECT) - 1) +
109+
1 + key_len + 1 + /* 'key' */
110+
1 /* ; */
111+
);
112+
113+
char *sql = malloc(sql_len + 1);
114+
SNFORMAT_S(sql, sql_len + 1, 4,
115+
SELECT, sizeof(SELECT) - 1,
116+
"'", (size_t) 1,
117+
key, key_len,
118+
"';", (size_t) 2);
119+
120+
str_t val;
121+
char *err = NULL;
122+
const int rc = sqlite3_exec(db, sql, ptkv_get_val, &val, &err);
123+
free(sql);
124+
125+
if (rc != SQLITE_OK) {
126+
sqlite3_result_error(context, err, -1);
127+
sqlite3_free(err);
128+
}
129+
130+
sqlite3_result_text(context, val.data, val.len, SQLITE_TRANSIENT);
131+
str_free_existing(&val);
132+
}

‎src/gufi_query/processdir.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ OF SUCH DAMAGE.
7575
#include "gufi_query/PoolArgs.h"
7676
#include "gufi_query/external.h"
7777
#include "gufi_query/gqw.h"
78+
#include "gufi_query/per_thread_kv.h"
7879
#include "gufi_query/process_queries.h"
7980
#include "gufi_query/processdir.h"
8081
#include "gufi_query/query.h"
@@ -146,7 +147,6 @@ static int collect_dir_inodes(void *args, int count, char **data, char **columns
146147
return 0;
147148
}
148149

149-
150150
int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
151151
/* Not checking arguments */
152152

@@ -218,6 +218,10 @@ int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
218218

219219
trie_insert(ta->user_strs, "n", 1, (void *) &n, NULL);
220220
trie_insert(ta->user_strs, "i", 1, (void *) &i, NULL);
221+
222+
ptkv_set_internal(id, db, "n", 1, n.data, n.len); /* ignore errors? */
223+
ptkv_set_internal(id, db, "i", 1, i.data, i.len); /* ignore errors? */
224+
221225
/* ********************************************** */
222226

223227
int recs = 1;

0 commit comments

Comments
 (0)