Skip to content

Commit 19bd85c

Browse files
calccryptocalccrypto
committed
switch templates to use in-memory buffers instead of copying from a file descriptor
also open with O_DIRECT
1 parent ff426f1 commit 19bd85c

File tree

4 files changed

+69
-76
lines changed

4 files changed

+69
-76
lines changed

‎include/template_db.h

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,13 +75,12 @@ extern "C" {
7575
#endif
7676

7777
struct template_db {
78-
int fd;
79-
off_t size;
78+
unsigned char *buf;
79+
sqlite3_int64 size;
8080
};
8181

8282
int init_template_db(struct template_db *tdb);
83-
int create_template(struct template_db *tdb, int (*create_tables)(const char *, sqlite3 *, void *),
84-
const char *name);
83+
int create_template(struct template_db *tdb, int (*create_tables)(const char *, sqlite3 *, void *));
8584

8685
int create_xattr_tables(const char *name, sqlite3 *db, void *args);
8786
int create_xattrs_template(struct template_db *tdb);

‎src/CMakeLists.txt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,18 @@ if (LOCALTIME_R)
6868
add_definitions(-DLOCALTIME_R)
6969
endif()
7070

71+
set(SAVED_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}")
72+
list(APPEND CMAKE_REQUIRED_DEFINITIONS "-D_GNU_SOURCE")
73+
check_symbol_exists(O_DIRECT "fcntl.h" HAVE_O_DIRECT)
74+
set(CMAKE_REQUIRED_DEFINITIONS "${SAVED_REQUIRED_DEFINITIONS}")
75+
76+
if (HAVE_O_DIRECT)
77+
message(STATUS "O_DIRECT flag is available")
78+
add_definitions(-DHAVE_O_DIRECT)
79+
else()
80+
message(STATUS "O_DIRECT flag is not available")
81+
endif()
82+
7183
# ##########################################
7284
# flags that can be used to modify code
7385
# behavior at compile time for some binaries.

‎src/template_db.c

Lines changed: 43 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,19 @@ OF SUCH DAMAGE.
6262

6363

6464

65+
#ifdef HAVE_O_DIRECT
66+
#ifndef _GNU_SOURCE
67+
#define _GNU_SOURCE /* O_DIRECT */
68+
#endif
69+
#endif
70+
6571
#include <errno.h>
6672
#include <fcntl.h>
6773
#include <stdio.h>
6874
#include <string.h>
6975
#include <sys/stat.h>
7076
#include <unistd.h>
7177

72-
#include "dbutils.h"
7378
#include "external.h"
7479
#include "template_db.h"
7580
#include "utils.h"
@@ -78,7 +83,7 @@ OF SUCH DAMAGE.
7883
int init_template_db(struct template_db *tdb) {
7984
/* Not checking argument */
8085

81-
tdb->fd = -1;
86+
tdb->buf = NULL;
8287
tdb->size = -1;
8388
return 0;
8489
}
@@ -112,75 +117,64 @@ int create_dbdb_tables(const char *name, sqlite3 *db, void *args) {
112117
int close_template_db(struct template_db *tdb) {
113118
/* Not checking argument */
114119

115-
close(tdb->fd);
120+
sqlite3_free(tdb->buf);
116121
return init_template_db(tdb);
117122
}
118123

119124
/* create the database file to copy from */
120-
int create_template(struct template_db *tdb, int (*create_tables)(const char *, sqlite3 *, void *),
121-
const char *name) {
122-
if (!tdb || (tdb->fd != -1) ||
123-
!create_tables || !name) {
125+
int create_template(struct template_db *tdb, int (*create_tables)(const char *, sqlite3 *, void *)) {
126+
if (!tdb || tdb->buf ||
127+
!create_tables) {
124128
return -1;
125129
}
126130

127-
sqlite3 *db = opendb(name, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0, 0, create_tables, NULL);
128-
129-
/*
130-
* open before sqlite3_close to prevent potential race
131-
* condition where file is deleted before being reopened
132-
*/
133-
tdb->fd = open(name, O_RDONLY);
134-
135-
sqlite3_close(db);
136-
137-
/* no need for the file to remain on the filesystem */
138-
remove(name);
139-
140-
if (tdb->fd == -1) {
141-
fprintf(stderr, "Could not open template file '%s'\n", name);
142-
return -1;
143-
}
144-
145-
tdb->size = lseek(tdb->fd, 0, SEEK_END);
146-
return !tdb->size;
131+
sqlite3 *db = opendb(SQLITE_MEMORY, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0, 0, create_tables, NULL);
132+
tdb->buf = sqlite3_serialize(db, "main", &tdb->size, 0);
133+
closedb(db);
134+
return !tdb->buf;
147135
}
148136

149137
/* create the initial xattrs database file to copy from */
150138
int create_xattrs_template(struct template_db *tdb) {
151-
static const char name[] = "xattrs_tmp.db";
152-
return create_template(tdb, create_xattr_tables, name);
139+
return create_template(tdb, create_xattr_tables);
153140
}
154141

155142
/* create the initial main database file to copy from */
156143
int create_dbdb_template(struct template_db *tdb) {
157-
static const char name[] = "tmp.db";
158-
return create_template(tdb, create_dbdb_tables, name);
144+
return create_template(tdb, create_dbdb_tables);
159145
}
160146

161147
/* copy the template file instead of creating a new database and new tables for each work item */
162148
/* the ownership and permissions are set too */
163149
int copy_template(struct template_db *tdb, const char *dst, uid_t uid, gid_t gid) {
164150
/* Not checking arguments */
165151

166-
int err = 0;
167-
const int src_db = dup(tdb->fd);
168-
err = err?err:errno;
169-
const int dst_db = open(dst, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);
170-
err = err?err:errno;
171-
const ssize_t sf = copyfd(src_db, 0, dst_db, 0, tdb->size);
172-
err = err?err:errno;
173-
fchown(dst_db, uid, gid);
174-
err = err?err:errno;
175-
close(src_db);
176-
err = err?err:errno;
177-
close(dst_db);
178-
err = err?err:errno;
179-
180-
if (sf == -1) {
181-
fprintf(stderr, "Could not copy template file (%d) to %s (%d): %s (%d)\n",
182-
src_db, dst, dst_db, strerror(err), err);
183-
remove(dst);
152+
const int fd = open(dst, O_WRONLY | O_CREAT
153+
#ifdef HAVE_O_DIRECT
154+
| O_DIRECT
155+
#endif
156+
, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);
157+
if (fd < 0) {
158+
return -1;
159+
}
160+
161+
ssize_t written = 0;
162+
while (written < tdb->size) {
163+
const ssize_t w = pwrite(fd, tdb->buf + written, tdb->size - written, 0);
164+
if (w < 1) {
165+
break;
166+
}
167+
168+
written += w;
169+
}
170+
171+
close(fd);
172+
173+
if (written != tdb->size) {
174+
return -1;
175+
}
176+
177+
if (chown(dst, uid, gid) != 0) {
184178
return -1;
185179
}
186180

‎test/unit/googletest/template_db.cpp

Lines changed: 11 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,6 @@ OF SUCH DAMAGE.
7676
#define TABLE_NAME "test_table"
7777
#define TABLE_CREATE "CREATE TABLE " TABLE_NAME "(A INT, B TEXT);"
7878

79-
static const char template_name[] = "should-not-see-this-in-filesystem";
80-
8179
static int sql_callback(void *args, int, char **data, char **) {
8280
std::string **cols = (std::string **) args;
8381
**cols = data[1];
@@ -99,15 +97,10 @@ TEST(template_db, create_copy) {
9997
struct template_db tdb;
10098
ASSERT_EQ(init_template_db(&tdb), 0);
10199

102-
ASSERT_EQ(create_template(&tdb, create_test_tables, template_name), 0);
103-
EXPECT_GT(tdb.fd, -1);
100+
ASSERT_EQ(create_template(&tdb, create_test_tables), 0);
101+
EXPECT_NE(tdb.buf, nullptr);
104102
EXPECT_GT(tdb.size, 0);
105103

106-
// the template file should disappear even
107-
// if it existed before this test
108-
struct stat st;
109-
EXPECT_EQ(lstat(template_name, &st), -1);
110-
111104
char dst_name[7];
112105
snprintf(dst_name, sizeof(dst_name), "XXXXXX");
113106
int dst = mkstemp(dst_name);
@@ -135,26 +128,21 @@ TEST(template_db, bad_inputs) {
135128
struct template_db tdb;
136129
ASSERT_EQ(init_template_db(&tdb), 0);
137130

138-
EXPECT_EQ(create_template(nullptr, create_test_tables, template_name), -1);
139-
EXPECT_EQ(create_template(&tdb, nullptr, template_name), -1);
140-
EXPECT_EQ(create_template(&tdb, create_test_tables, nullptr), -1);
141-
EXPECT_EQ(create_template(&tdb, create_test_tables, ""), -1);
142-
143-
tdb.fd = 0;
144-
EXPECT_EQ(create_template(&tdb, create_test_tables, template_name), -1);
145-
tdb.fd = -1;
131+
EXPECT_EQ(create_template(nullptr, create_test_tables), -1);
132+
EXPECT_EQ(create_template(&tdb, nullptr), -1);
133+
EXPECT_EQ(create_template(&tdb, create_test_tables), -1);
146134

147-
EXPECT_EQ(copy_template(&tdb, template_name, -1, -1), -1);
135+
EXPECT_EQ(copy_template(&tdb, "", -1, -1), -1);
148136

149-
EXPECT_EQ(template_to_db(&tdb, template_name, -1, -1), nullptr);
137+
EXPECT_EQ(template_to_db(&tdb, "", -1, -1), nullptr);
150138

151139
ASSERT_EQ(close_template_db(&tdb), 0);
152140
}
153141

154142
TEST(create_empty_dbdb, good) {
155143
struct template_db tdb;
156144
ASSERT_EQ(init_template_db(&tdb), 0);
157-
ASSERT_EQ(create_template(&tdb, create_test_tables, template_name), 0);
145+
ASSERT_EQ(create_template(&tdb, create_test_tables), 0);
158146

159147
// create new directory
160148
char dirname[] = "XXXXXX" ;
@@ -198,10 +186,10 @@ TEST(create_empty_dbdb, good) {
198186
EXPECT_EQ(unlink(dbname), 0);
199187

200188
// try to create db with a bad file descriptor
201-
const int tfd = tdb.fd;
202-
tdb.fd = -1;
189+
unsigned char *tbuf = tdb.buf;
190+
tdb.buf = nullptr;
203191
EXPECT_EQ(create_empty_dbdb(&tdb, &dst, -1, -1), -1);
204-
tdb.fd = tfd;
192+
tdb.buf = tbuf;
205193

206194
// create new db
207195
EXPECT_EQ(create_empty_dbdb(&tdb, &dst, -1, -1), 0);

0 commit comments

Comments
 (0)