Skip to content

Commit 7c0ffcb

Browse files
authored
Merge pull request #357 from mbezhanov/libsql-support
Add LibSQL support to SQLite generator
2 parents 5d6bdfc + 1ec7c16 commit 7c0ffcb

File tree

10 files changed

+548
-29
lines changed

10 files changed

+548
-29
lines changed

.github/workflows/test.yml

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,28 @@ jobs:
3232
&& mysql --execute 'CREATE DATABASE dialect_droppable;'
3333
&& mysql --execute 'CREATE DATABASE driver_droppable;'
3434
35+
- name: LibSQL Setup
36+
run: |
37+
mkdir -p ./regexp \
38+
&& wget https://github.com/nalgeon/sqlean/releases/download/0.27.1/sqlean-linux-x86.zip \
39+
&& unzip sqlean-linux-x86.zip regexp.so \
40+
&& rm sqlean-linux-x86.zip \
41+
&& sha256sum regexp.so > ./regexp/trusted.lst \
42+
&& mv regexp.so ./regexp \
43+
&& docker run -d -p 8080:8080 -p 8000:8000 --rm -v ${{ github.workspace }}/regexp:/var/lib/sqld/regexp \
44+
ghcr.io/tursodatabase/libsql-server:c6e4e09 sqld \
45+
--http-listen-addr=0.0.0.0:8080 \
46+
--admin-listen-addr=0.0.0.0:8000 \
47+
--enable-namespaces \
48+
--extensions-path /var/lib/sqld/regexp \
49+
&& sleep 1 \
50+
&& curl -X POST http://localhost:8000/v1/namespaces/one/create \
51+
-H "Content-Type: application/json" \
52+
--data '{}' \
53+
&& curl -X POST http://localhost:8000/v1/namespaces/one/config \
54+
-H "Content-Type: application/json" \
55+
--data '{"block_reads": false, "block_writes": false, "block_reason": null, "max_db_size": "1000.0 PB", "heartbeat_url": null, "jwt_key": null, "allow_attach": true, "txn_timeout_s": null, "durability_mode": "relaxed"}'
56+
3557
- name: Checkout Repo
3658
uses: actions/checkout@v4
3759

@@ -66,12 +88,12 @@ jobs:
6688

6789
test-windows-sqlite:
6890
# Run generation test on windows to catch filepath issues
69-
# Testing Postgres and MySQL are not possible since the windows runner
91+
# Testing Postgres, MySQL, and LibSQL are not possible since the windows runner
7092
# does not support containers
7193
runs-on: windows-latest
7294
strategy:
7395
matrix:
74-
go: ['stable' ]
96+
go: ['stable']
7597
steps:
7698
- name: Checkout Repo
7799
uses: actions/checkout@v4
@@ -85,4 +107,4 @@ jobs:
85107
run: go mod download
86108

87109
- name: Run tests
88-
run: go test -race ./gen/bobgen-sqlite/driver
110+
run: go test -race -run '^(TestAssembleSQLite)$' ./gen/bobgen-sqlite/driver

gen/bobgen-sqlite/driver/sqlite.go

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,17 @@ import (
55
"database/sql"
66
"errors"
77
"fmt"
8+
"net/url"
89
"sort"
10+
"strconv"
911
"strings"
1012

1113
"github.com/aarondl/opt/null"
1214
helpers "github.com/stephenafamo/bob/gen/bobgen-helpers"
1315
"github.com/stephenafamo/bob/gen/drivers"
1416
"github.com/stephenafamo/scan"
1517
"github.com/stephenafamo/scan/stdscan"
18+
_ "github.com/tursodatabase/libsql-client-go/libsql"
1619
"github.com/volatiletech/strmangle"
1720
_ "modernc.org/sqlite"
1821
)
@@ -93,14 +96,18 @@ func (d *driver) Assemble(ctx context.Context) (*DBInfo, error) {
9396
return nil, fmt.Errorf("database dsn is not set")
9497
}
9598

96-
d.conn, err = sql.Open("sqlite", d.config.DSN)
99+
driverName := d.inferDriver()
100+
d.conn, err = sql.Open(driverName, d.config.DSN)
97101
if err != nil {
98102
return nil, fmt.Errorf("failed to connect to database: %w", err)
99103
}
100104
defer d.conn.Close()
101105

102106
for schema, dsn := range d.config.Attach {
103-
_, err = d.conn.ExecContext(ctx, fmt.Sprintf("attach database '%s' as %s", dsn, schema))
107+
if driverName == "sqlite" {
108+
dsn = strconv.Quote(dsn)
109+
}
110+
_, err = d.conn.ExecContext(ctx, fmt.Sprintf("attach database %s as %s", dsn, schema))
104111
if err != nil {
105112
return nil, fmt.Errorf("could not attach %q: %w", schema, err)
106113
}
@@ -111,6 +118,9 @@ func (d *driver) Assemble(ctx context.Context) (*DBInfo, error) {
111118
return nil, err
112119
}
113120

121+
if driverName == "libsql" {
122+
d.config.DriverName = "github.com/tursodatabase/libsql-client-go/libsql"
123+
}
114124
dbinfo := &DBInfo{
115125
DriverName: d.config.DriverName,
116126
Tables: tables,
@@ -119,6 +129,29 @@ func (d *driver) Assemble(ctx context.Context) (*DBInfo, error) {
119129
return dbinfo, nil
120130
}
121131

132+
func (d *driver) inferDriver() string {
133+
driverName := "sqlite"
134+
if !strings.Contains(d.config.DSN, "://") {
135+
return driverName
136+
}
137+
dsn, _ := url.Parse(d.config.DSN)
138+
if dsn == nil {
139+
return driverName
140+
}
141+
libsqlSchemes := map[string]bool{
142+
"libsql": true,
143+
"file": true,
144+
"https": true,
145+
"http": true,
146+
"wss": true,
147+
"ws": true,
148+
}
149+
if libsqlSchemes[dsn.Scheme] {
150+
driverName = "libsql"
151+
}
152+
return driverName
153+
}
154+
122155
func (d *driver) buildQuery(schema string) (string, []any) {
123156
var args []any
124157
query := fmt.Sprintf(`SELECT name FROM %q.sqlite_schema WHERE name NOT LIKE 'sqlite_%%' AND type IN ('table', 'view')`, schema)
@@ -135,13 +168,13 @@ func (d *driver) buildQuery(schema string) (string, []any) {
135168
}
136169
}
137170
if len(include) > 0 {
138-
subqueries = append(subqueries, fmt.Sprintf("name in (%s)", strmangle.Placeholders(true, len(include), 1, 1)))
171+
subqueries = append(subqueries, fmt.Sprintf("name in (%s)", strmangle.Placeholders(false, len(include), 1, 1)))
139172
for _, w := range include {
140173
args = append(args, w)
141174
}
142175
}
143176
if len(regexPatterns) > 0 {
144-
subqueries = append(subqueries, fmt.Sprintf("name regexp (%s)", strmangle.Placeholders(true, 1, len(args)+1, 1)))
177+
subqueries = append(subqueries, fmt.Sprintf("name regexp (%s)", strmangle.Placeholders(false, 1, len(args)+1, 1)))
145178
args = append(args, strings.Join(regexPatterns, "|"))
146179
}
147180
if len(subqueries) > 0 {
@@ -159,13 +192,13 @@ func (d *driver) buildQuery(schema string) (string, []any) {
159192
}
160193
}
161194
if len(exclude) > 0 {
162-
subqueries = append(subqueries, fmt.Sprintf("name not in (%s)", strmangle.Placeholders(true, len(exclude), 1+len(args), 1)))
195+
subqueries = append(subqueries, fmt.Sprintf("name not in (%s)", strmangle.Placeholders(false, len(exclude), 1+len(args), 1)))
163196
for _, w := range exclude {
164197
args = append(args, w)
165198
}
166199
}
167200
if len(regexPatterns) > 0 {
168-
subqueries = append(subqueries, fmt.Sprintf("name not regexp (%s)", strmangle.Placeholders(true, 1, len(args)+1, 1)))
201+
subqueries = append(subqueries, fmt.Sprintf("name not regexp (%s)", strmangle.Placeholders(false, 1, len(args)+1, 1)))
169202
args = append(args, strings.Join(regexPatterns, "|"))
170203
}
171204
if len(subqueries) > 0 {

gen/bobgen-sqlite/driver/sqlite_test.go

Lines changed: 156 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
package driver
22

33
import (
4+
"bytes"
45
"context"
56
"database/sql"
67
sqlDriver "database/sql/driver"
7-
_ "embed"
8+
"embed"
89
"errors"
910
"flag"
1011
"fmt"
1112
"io/fs"
1213
"os"
1314
"regexp"
15+
"strconv"
16+
"strings"
1417
"testing"
1518

1619
"github.com/stephenafamo/bob/gen"
@@ -21,7 +24,7 @@ import (
2124
"modernc.org/sqlite"
2225
)
2326

24-
func cleanup(t *testing.T, config Config) {
27+
func cleanupSQLite(t *testing.T, config Config) {
2528
t.Helper()
2629

2730
fmt.Printf("cleaning...")
@@ -40,42 +43,131 @@ func cleanup(t *testing.T, config Config) {
4043
fmt.Printf(" DONE\n")
4144
}
4245

46+
func cleanupLibSQL(t *testing.T, db *sql.DB) {
47+
t.Helper()
48+
49+
fmt.Printf("cleaning...")
50+
51+
// Find all tables
52+
rows, err := db.Query("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';")
53+
if err != nil {
54+
t.Fatal(err)
55+
}
56+
defer rows.Close()
57+
58+
// Drop each table
59+
for rows.Next() {
60+
var tableName string
61+
if err = rows.Scan(&tableName); err != nil {
62+
t.Fatalf("could not delete existing db: %v", err)
63+
}
64+
_, err = db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %q;", tableName))
65+
if err != nil {
66+
t.Fatalf("could not delete %q table: %v", tableName, err)
67+
}
68+
}
69+
70+
fmt.Printf(" DONE\n")
71+
}
72+
4373
var flagOverwriteGolden = flag.Bool("overwrite-golden", false, "Overwrite the golden file with the current execution results")
4474

45-
func TestAssemble(t *testing.T) {
75+
func TestAssembleSQLite(t *testing.T) {
4676
ctx := context.Background()
4777

4878
config := Config{
4979
DSN: "./test.db",
5080
Attach: map[string]string{"one": "./test1.db"},
5181
}
5282

53-
cleanup(t, config)
54-
t.Cleanup(func() { cleanup(t, config) })
83+
cleanupSQLite(t, config)
84+
t.Cleanup(func() { cleanupSQLite(t, config) })
5585

56-
db, err := sql.Open("sqlite", config.DSN)
57-
if err != nil {
58-
t.Fatalf("failed to connect to database: %v", err)
59-
}
86+
db := connect(t, "sqlite", config.DSN)
6087
defer db.Close()
6188

62-
if err = registerRegexpFunction(); err != nil {
89+
if err := registerRegexpFunction(); err != nil {
90+
t.Fatal(err)
91+
}
92+
93+
attach(t, ctx, db, config)
94+
95+
fmt.Printf("migrating...")
96+
migrate(t, db, testfiles.SQLiteSchema)
97+
fmt.Printf(" DONE\n")
98+
99+
assemble(t, config)
100+
}
101+
102+
func TestAssembleLibSQL(t *testing.T) {
103+
ctx := context.Background()
104+
105+
err := adjustGoldenFiles()
106+
if err != nil {
63107
t.Fatal(err)
64108
}
65109

110+
config := Config{
111+
DSN: "ws://localhost:8080",
112+
Attach: map[string]string{"one": "one"},
113+
}
114+
115+
db := connect(t, "libsql", config.DSN)
116+
117+
attach(t, ctx, db, config)
118+
119+
fmt.Printf("migrating...")
120+
dbHttpDefault := connect(t, "libsql", "http://localhost:8080")
121+
migrate(t, dbHttpDefault, testfiles.LibSQLDefaultSchema)
122+
dbHttpOne := connect(t, "libsql", "http://one.localhost:8080")
123+
migrate(t, dbHttpOne, testfiles.LibSQLOneSchema)
124+
fmt.Printf(" DONE\n")
125+
126+
t.Cleanup(func() {
127+
cleanupLibSQL(t, dbHttpDefault)
128+
cleanupLibSQL(t, dbHttpOne)
129+
dbHttpDefault.Close()
130+
dbHttpOne.Close()
131+
err = restoreGoldenFiles()
132+
if err != nil {
133+
t.Fatal(err)
134+
}
135+
})
136+
137+
assemble(t, config)
138+
}
139+
140+
func connect(t *testing.T, driverName, dsn string) *sql.DB {
141+
t.Helper()
142+
db, err := sql.Open(driverName, dsn)
143+
if err != nil {
144+
t.Fatalf("failed to connect to database: %v", err)
145+
}
146+
return db
147+
}
148+
149+
func attach(t *testing.T, ctx context.Context, db *sql.DB, config Config) {
150+
t.Helper()
66151
for schema, conn := range config.Attach {
67-
_, err = db.ExecContext(ctx, fmt.Sprintf("attach database '%s' as %q", conn, schema))
152+
if strings.HasPrefix(conn, "./") {
153+
conn = strconv.Quote(conn)
154+
}
155+
_, err := db.ExecContext(ctx, fmt.Sprintf("attach database %s as %s", conn, schema))
68156
if err != nil {
69157
t.Fatalf("could not attach %q: %v", conn, err)
70158
}
71159
}
160+
}
72161

73-
fmt.Printf("migrating...")
74-
if err := helpers.Migrate(context.Background(), db, testfiles.SQLiteSchema); err != nil {
162+
func migrate(t *testing.T, db *sql.DB, schema embed.FS) {
163+
t.Helper()
164+
if err := helpers.Migrate(context.Background(), db, schema); err != nil {
75165
t.Fatal(err)
76166
}
77-
fmt.Printf(" DONE\n")
167+
}
78168

169+
func assemble(t *testing.T, config Config) {
170+
t.Helper()
79171
tests := []struct {
80172
name string
81173
config Config
@@ -243,3 +335,53 @@ func registerRegexpFunction() error {
243335
return match, nil
244336
})
245337
}
338+
339+
var goldenFiles = []string{
340+
"exclude-tables.golden.json",
341+
"include-exclude-tables.golden.json",
342+
"include-exclude-tables-mixed.golden.json",
343+
"include-exclude-tables-regex.golden.json",
344+
"include-tables.golden.json",
345+
"sqlite.golden.json",
346+
}
347+
348+
func adjustGoldenFiles() error {
349+
for _, f := range goldenFiles {
350+
err := replaceStringInFile(f, "modernc.org/sqlite", "github.com/tursodatabase/libsql-client-go/libsql")
351+
if err != nil {
352+
return err
353+
}
354+
}
355+
return nil
356+
}
357+
358+
func restoreGoldenFiles() error {
359+
for _, f := range goldenFiles {
360+
err := replaceStringInFile(f, "github.com/tursodatabase/libsql-client-go/libsql", "modernc.org/sqlite")
361+
if err != nil {
362+
return err
363+
}
364+
}
365+
return nil
366+
}
367+
368+
func replaceStringInFile(filename, oldStr, newStr string) error {
369+
fileInfo, err := os.Stat(filename)
370+
if err != nil {
371+
return err
372+
}
373+
perm := fileInfo.Mode().Perm()
374+
375+
input, err := os.ReadFile(filename)
376+
if err != nil {
377+
return err
378+
}
379+
output := bytes.ReplaceAll(input, []byte(oldStr), []byte(newStr))
380+
381+
err = os.WriteFile(filename, output, perm)
382+
if err != nil {
383+
return err
384+
}
385+
386+
return nil
387+
}

0 commit comments

Comments
 (0)