Skip to content

Commit

Permalink
Merge pull request #1848 from tursodatabase/wal-sync-tests-2
Browse files Browse the repository at this point in the history 
refactor wal sync tests
  • Loading branch information
@penberg
penberg authored Nov 28, 2024
2 parents 8b1c943 + 98ce757 commit f453e44
Showing 1 changed file with 70 additions and 105 deletions.
175 changes: 70 additions & 105 deletions libsql-sqlite3/src/test_walapi.c
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
#include <sqlite3.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <assert.h>

#define eprintf(...) fprintf(stderr, __VA_ARGS__)
#define ensure(condition, ...) { if (!(condition)) { eprintf(__VA_ARGS__); exit(1); } }

#if 0
static void dump_frame(unsigned char *frame, size_t size){
for(int addr=0; addr<size; addr+=16){
Expand All @@ -26,140 +31,100 @@ static void dump_frame(unsigned char *frame, size_t size){
}
#endif

static int cmp_data(sqlite3 *db1, sqlite3 *db2){
static void cmp_data(sqlite3 *db1, sqlite3 *db2){
sqlite3_stmt *stmt1, *stmt2;
int rc;

rc = sqlite3_prepare_v2(db1, "SELECT HEX(x) FROM t", -1, &stmt1, 0);
if( rc!=SQLITE_OK ){
fprintf(stderr, "Can't prepare statement: %s\n", sqlite3_errmsg(db1));
return 1;
}

rc = sqlite3_prepare_v2(db2, "SELECT HEX(x) FROM t", -1, &stmt2, 0);
if( rc!=SQLITE_OK ){
fprintf(stderr, "Can't prepare statement: %s\n", sqlite3_errmsg(db2));
return 1;
}
ensure(sqlite3_prepare_v2(db1, "SELECT HEX(x) FROM t", -1, &stmt1, 0) == SQLITE_OK, "can't prepare statement: %s\n", sqlite3_errmsg(db1));
ensure(sqlite3_prepare_v2(db2, "SELECT HEX(x) FROM t", -1, &stmt2, 0) == SQLITE_OK, "can't prepare statement: %s\n", sqlite3_errmsg(db2));

for(;;){
int step1 = sqlite3_step(stmt1);
int step2 = sqlite3_step(stmt2);
if( step1!=step2 ){
fprintf(stderr, "Step mismatch: %d != %d\n", step1, step2);
return 1;
}
ensure(step1 == step2, "step mismatch: %d != %d\n", step1, step2);
if( step1!=SQLITE_ROW ){
break;
}
const unsigned char *text1 = sqlite3_column_text(stmt1, 0);
const unsigned char *text2 = sqlite3_column_text(stmt2, 0);
if( strncmp((const char *)text1, (const char *)text2, 4096)!=0 ){
fprintf(stderr, "Data mismatch\n");
return 1;
}
ensure(strncmp((const char *)text1, (const char *)text2, 4096) == 0, "data mismatch");
}
return 0;
}

static int sync_db(sqlite3 *db_primary, sqlite3 *db_backup){
static void sync_db(sqlite3 *db_primary, sqlite3 *db_backup){
unsigned int max_frame;
int rc;

rc = libsql_wal_frame_count(db_primary, &max_frame);
if( rc!=SQLITE_OK ){
fprintf(stderr, "Can't get frame count: %s\n", sqlite3_errmsg(db_primary));
return 1;
}
rc = libsql_wal_insert_begin(db_backup);
if( rc!=SQLITE_OK ){
fprintf(stderr, "Can't begin commit: %s\n", sqlite3_errmsg(db_backup));
return 1;
}
ensure(libsql_wal_frame_count(db_primary, &max_frame) == SQLITE_OK, "can't get frame count: %s\n", sqlite3_errmsg(db_primary));
ensure(libsql_wal_insert_begin(db_backup) == SQLITE_OK, "can't begin commit: %s\n", sqlite3_errmsg(db_backup));
for(int i=1; i<=max_frame; i++){
char frame[4096+24];
rc = libsql_wal_get_frame(db_primary, i, frame, sizeof(frame));
if( rc!=SQLITE_OK ){
fprintf(stderr, "Can't get frame: %s\n", sqlite3_errmsg(db_primary));
return 1;
}
rc = libsql_wal_insert_frame(db_backup, i, frame, sizeof(frame));
if( rc!=SQLITE_OK ){
fprintf(stderr, "Can't inject frame %d: %s\n", rc, sqlite3_errmsg(db_backup));
return 1;
}
}
rc = libsql_wal_insert_end(db_backup);
if( rc!=SQLITE_OK ){
fprintf(stderr, "Can't end commit: %s\n", sqlite3_errmsg(db_backup));
return 1;
ensure(libsql_wal_get_frame(db_primary, i, frame, sizeof(frame)) == SQLITE_OK, "can't get frame: %s\n", sqlite3_errmsg(db_primary));
ensure(libsql_wal_insert_frame(db_backup, i, frame, sizeof(frame)) == SQLITE_OK, "can't inject frame: %s\n", sqlite3_errmsg(db_backup));
}
return 0;
}

static void gen_data(sqlite3 *db){
sqlite3_exec(db, "CREATE TABLE t (x)", 0, 0, 0);
sqlite3_exec(db, "INSERT INTO t VALUES (randomblob(4 * 1024))", 0, 0, 0);
sqlite3_exec(db, "INSERT INTO t VALUES (randomblob(1 * 1024))", 0, 0, 0);
sqlite3_exec(db, "INSERT INTO t VALUES (randomblob(1 * 1024))", 0, 0, 0);
ensure(libsql_wal_insert_end(db_backup) == SQLITE_OK, "can't end commit: %s\n", sqlite3_errmsg(db_backup));
}

int open_db(const char *path, sqlite3 **db) {
int rc;

rc = sqlite3_open(path, db);
if (rc != SQLITE_OK) {
fprintf(stderr, "Can't open database %s: %s\n", path, sqlite3_errmsg(*db));
return rc;
}
rc = sqlite3_exec(*db, "PRAGMA journal_mode=WAL", NULL, NULL, NULL);
if (rc != SQLITE_OK) {
fprintf(stderr, "Can't set journal mode for %s: %s\n", path, sqlite3_errmsg(*db));
return rc;
}
rc = sqlite3_wal_autocheckpoint(*db, 0);
if (rc != SQLITE_OK) {
fprintf(stderr, "Can't disable checkpointing for %s: %s\n", path, sqlite3_errmsg(*db));
return rc;
}
return rc;
void open_db(const char *path, sqlite3 **db) {
ensure(sqlite3_open(path, db) == SQLITE_OK, "can't open database %s: %s\n", path, sqlite3_errmsg(*db));
ensure(sqlite3_exec(*db, "PRAGMA journal_mode=WAL", NULL, NULL, NULL) == SQLITE_OK, "can't set journal mode for %s: %s\n", path, sqlite3_errmsg(*db));
ensure(sqlite3_wal_autocheckpoint(*db, 0) == SQLITE_OK, "can't disable checkpointing for %s: %s\n", path, sqlite3_errmsg(*db));
}

int main(int argc, char *argv[])
{
void test_huge_payload() {
sqlite3 *db_primary, *db_backup;
unsigned int max_frame;
int rc;

rc = open_db("primary.db", &db_primary);
if (rc != SQLITE_OK) {
return 1;
}
gen_data(db_primary);

rc = open_db("backup.db", &db_backup);
if (rc != SQLITE_OK) {
return 1;
}
rc = libsql_wal_frame_count(db_backup, &max_frame);
if (rc != SQLITE_OK) {
fprintf(stderr, "Can't get frame count for backup: %s\n", sqlite3_errmsg(db_backup));
return 1;
}
open_db("primary_test_huge_payload.db", &db_primary);
ensure(sqlite3_exec(db_primary, "CREATE TABLE t (x)", 0, 0, 0) == SQLITE_OK, "failed to insert data\n");
ensure(sqlite3_exec(db_primary, "INSERT INTO t VALUES (randomblob(4 * 1024))", 0, 0, 0) == SQLITE_OK, "failed to insert data\n");
ensure(sqlite3_exec(db_primary, "INSERT INTO t VALUES (randomblob(1 * 1024))", 0, 0, 0) == SQLITE_OK, "failed to insert data\n");
ensure(sqlite3_exec(db_primary, "INSERT INTO t VALUES (randomblob(1 * 1024))", 0, 0, 0) == SQLITE_OK, "failed to insert data\n");

open_db("backup_test_huge_payload.db", &db_backup);
ensure(libsql_wal_frame_count(db_backup, &max_frame) == SQLITE_OK, "failed to get frames count: %s\n", sqlite3_errmsg(db_backup));
assert(max_frame == 0);

eprintf("start full sync\n");
sync_db(db_primary, db_backup);

if (cmp_data(db_primary, db_backup)) {
return 1;
}

cmp_data(db_primary, db_backup);
sync_db(db_primary, db_backup);
if (cmp_data(db_primary, db_backup)) {
return 1;
cmp_data(db_primary, db_backup);
}

void test_sync_by_parts() {
sqlite3 *db_primary, *db_backup;
unsigned int max_frame;
uint32_t in_commit = 0;
open_db("primary_test_sync_by_parts.db", &db_primary);
ensure(sqlite3_exec(db_primary, "CREATE TABLE t (x)", 0, 0, 0) == SQLITE_OK, "failed to insert data\n");
ensure(sqlite3_exec(db_primary, "INSERT INTO t VALUES (randomblob(4 * 1024))", 0, 0, 0) == SQLITE_OK, "failed to insert data\n");
ensure(sqlite3_exec(db_primary, "INSERT INTO t VALUES (randomblob(1 * 1024))", 0, 0, 0) == SQLITE_OK, "failed to insert data\n");
ensure(sqlite3_exec(db_primary, "INSERT INTO t VALUES (randomblob(1 * 1024))", 0, 0, 0) == SQLITE_OK, "failed to insert data\n");

open_db("backup_test_sync_by_parts.db", &db_backup);

ensure(libsql_wal_frame_count(db_primary, &max_frame) == SQLITE_OK, "can't get frame count: %s\n", sqlite3_errmsg(db_primary));
eprintf("start sync by parts\n");
for(int i=1; i<=max_frame; i++){
char frame[4096+24];
uint32_t is_commit;
ensure(libsql_wal_get_frame(db_primary, i, frame, sizeof(frame)) == SQLITE_OK, "can't get frame: %s\n", sqlite3_errmsg(db_primary));
is_commit = ((uint32_t)frame[4] << 24) + ((uint32_t)frame[5] << 16) + ((uint32_t)frame[6] << 8) + ((uint32_t)frame[7] << 0);
if (!in_commit) {
in_commit = 1;
ensure(libsql_wal_insert_begin(db_backup) == SQLITE_OK, "can't begin commit: %s\n", sqlite3_errmsg(db_backup));
}
ensure(libsql_wal_insert_frame(db_backup, i, frame, sizeof(frame)) == SQLITE_OK, "can't inject frame: %s\n", sqlite3_errmsg(db_backup));
if (is_commit) {
ensure(libsql_wal_insert_end(db_backup) == SQLITE_OK, "can't end commit: %s\n", sqlite3_errmsg(db_backup));
in_commit = 0;
}
}
cmp_data(db_primary, db_backup);
}

printf("OK\n");
int main(int argc, char *argv[])
{
test_huge_payload();
printf("============= OK test_huge_payload\n");

test_sync_by_parts();
printf("============= OK test_sync_by_parts\n");
return 0;
}

0 comments on commit f453e44

Please sign in to comment.