Skip to content

tapdb: implement basic migration downgrade protection #973

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 26, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .github/workflows/main.yaml
Original file line number Diff line number Diff line change
@@ -81,6 +81,16 @@ jobs:
- name: Run test vector creation check
run: make test-vector-check

migration-version-check:
name: migration version check
runs-on: ubuntu-latest
steps:
- name: git checkout
uses: actions/checkout@v3

- name: Run migration version check
run: make migration-version-check

########################
# Compilation check.
########################
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -323,6 +323,10 @@ test-vector-check: gen-deterministic-test-vectors
@$(call print, "Checking deterministic test vectors.")
if test -n "$$(git status | grep -e ".json")"; then echo "Test vectors not updated"; git status; git diff; exit 1; fi

migration-version-check:
@$(call print, "Checking migration version.")
./scripts/check-migration-latest-version.sh

clean:
@$(call print, "Cleaning source.$(NC)")
$(RM) coverage.txt
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
@@ -49,7 +49,7 @@ require (
google.golang.org/protobuf v1.33.0
gopkg.in/macaroon-bakery.v2 v2.1.0
gopkg.in/macaroon.v2 v2.1.0
modernc.org/sqlite v1.29.8
modernc.org/sqlite v1.30.0
)

require (
@@ -195,7 +195,7 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
modernc.org/libc v1.49.3 // indirect
modernc.org/libc v1.50.9 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.8.0 // indirect
modernc.org/strutil v1.2.0 // indirect
18 changes: 8 additions & 10 deletions go.sum
Original file line number Diff line number Diff line change
@@ -525,8 +525,6 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
@@ -1143,18 +1141,18 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
modernc.org/cc/v4 v4.20.0 h1:45Or8mQfbUqJOG9WaxvlFYOAQO0lQ5RvqBcFCXngjxk=
modernc.org/cc/v4 v4.20.0/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
modernc.org/ccgo/v4 v4.16.0 h1:ofwORa6vx2FMm0916/CkZjpFPSR70VwTjUCe2Eg5BnA=
modernc.org/ccgo/v4 v4.16.0/go.mod h1:dkNyWIjFrVIZ68DTo36vHK+6/ShBn4ysU61So6PIqCI=
modernc.org/cc/v4 v4.21.2 h1:dycHFB/jDc3IyacKipCNSDrjIC0Lm1hyoWOZTRR20Lk=
modernc.org/cc/v4 v4.21.2/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
modernc.org/ccgo/v4 v4.17.8 h1:yyWBf2ipA0Y9GGz/MmCmi3EFpKgeS7ICrAFes+suEbs=
modernc.org/ccgo/v4 v4.17.8/go.mod h1:buJnJ6Fn0tyAdP/dqePbrrvLyr6qslFfTbFrCuaYvtA=
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
modernc.org/libc v1.49.3 h1:j2MRCRdwJI2ls/sGbeSk0t2bypOG/uvPZUsGQFDulqg=
modernc.org/libc v1.49.3/go.mod h1:yMZuGkn7pXbKfoT/M35gFJOAEdSKdxL0q64sF7KqCDo=
modernc.org/libc v1.50.9 h1:hIWf1uz55lorXQhfoEoezdUHjxzuO6ceshET/yWjSjk=
modernc.org/libc v1.50.9/go.mod h1:15P6ublJ9FJR8YQCGy8DeQ2Uwur7iW9Hserr/T3OFZE=
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
@@ -1163,8 +1161,8 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
modernc.org/sqlite v1.29.8 h1:nGKglNx9K5v0As+zF0/Gcl1kMkmaU1XynYyq92PbsC8=
modernc.org/sqlite v1.29.8/go.mod h1:lQPm27iqa4UNZpmr4Aor0MH0HkCLbt1huYDfWylLZFk=
modernc.org/sqlite v1.30.0 h1:8YhPUs/HTnlEgErn/jSYQTwHN/ex8CjHHjg+K9iG7LM=
modernc.org/sqlite v1.30.0/go.mod h1:cgkTARJ9ugeXSNaLBPK3CqbOe7Ec7ZhWPoMFGldEYEw=
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
22 changes: 22 additions & 0 deletions scripts/check-migration-latest-version.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/bash

set -e

# Get the latest version number from the migration file names.
migrations_path="tapdb/sqlc/migrations"
latest_file_version=$(ls -r $migrations_path | grep .up.sql | head -1 | cut -d_ -f1)

# Force base 10 interpretation, getting rid of the leading zeroes.
latest_file_version=$((10#$latest_file_version))

# Check the value in migrations.go.
file_path="tapdb/migrations.go"
latest_code_version=$(grep -oP 'LatestMigrationVersion\s*=\s*\K\d+' "$file_path")

if [ "$latest_file_version" -ne "$latest_code_version" ]; then
echo "Latest migration version in file names: $latest_file_version"
echo "Latest migration version in code: $latest_code_version"
exit 1
fi

echo "Latest migration version in file names and code match: $latest_file_version"
53 changes: 52 additions & 1 deletion tapdb/migrations.go
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ package tapdb
import (
"bytes"
"errors"
"fmt"
"io"
"io/fs"
"net/http"
@@ -12,6 +13,16 @@ import (
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database"
"github.com/golang-migrate/migrate/v4/source/httpfs"
"github.com/lightninglabs/taproot-assets/fn"
)

const (
// LatestMigrationVersion is the latest migration version of the
// database. This is used to implement downgrade protection for the
// daemon.
//
// NOTE: This MUST be updated when a new migration is added.
LatestMigrationVersion = 21
)

// MigrationTarget is a functional option that can be passed to applyMigrations
@@ -34,6 +45,36 @@ var (
}
)

var (
// ErrMigrationDowngrade is returned when a database downgrade is
// detected.
ErrMigrationDowngrade = errors.New("database downgrade detected")
)

// migrationOption is a functional option that can be passed to migrate related
// methods to modify their behavior.
type migrateOptions struct {
latestVersion fn.Option[uint]
}

// defaultMigrateOptions returns a new migrateOptions instance with default
// settings.
func defaultMigrateOptions() *migrateOptions {
return &migrateOptions{}
}

// MigrateOpt is a functional option that can be passed to migrate related
// methods to modify behavior.
type MigrateOpt func(*migrateOptions)

// WithLatestVersion allows callers to override the default latest version
// setting.
func WithLatestVersion(version uint) MigrateOpt {
return func(o *migrateOptions) {
o.latestVersion = fn.Some(version)
}
}

// migrationLogger is a logger that wraps the passed btclog.Logger so it can be
// used to log migrations.
type migrationLogger struct {
@@ -72,7 +113,7 @@ func (m *migrationLogger) Verbose() bool {
// system under the given path, using the passed database driver and database
// name, up to or down to the given target version.
func applyMigrations(fs fs.FS, driver database.Driver, path, dbName string,
targetVersion MigrationTarget) error {
targetVersion MigrationTarget, opts *migrateOptions) error {

// With the migrate instance open, we'll create a new migration source
// using the embedded file system stored in sqlSchemas. The library
@@ -95,6 +136,16 @@ func applyMigrations(fs fs.FS, driver database.Driver, path, dbName string,

migrationVersion, _, _ := sqlMigrate.Version()

// As the down migrations may end up *dropping* data, we want to
// prevent that without explicit accounting.
latestVersion := opts.latestVersion.UnwrapOr(LatestMigrationVersion)
if migrationVersion > latestVersion {
return fmt.Errorf("%w: database version is newer than the "+
"latest migration version, preventing downgrade: "+
"db_version=%v, latest_migration_version=%v",
ErrMigrationDowngrade, migrationVersion, latestVersion)
}

log.Infof("Applying migrations from version=%v", migrationVersion)

// Apply our local logger to the migration instance.
12 changes: 12 additions & 0 deletions tapdb/migrations_test.go
Original file line number Diff line number Diff line change
@@ -118,3 +118,15 @@ func TestMigration15(t *testing.T) {
t, wire.TxWitness{{0xbb}}, assets[0].PrevWitnesses[1].TxWitness,
)
}

// TestMigrationDowngrade tests that downgrading the database is prevented.
func TestMigrationDowngrade(t *testing.T) {
// For this test, with the current hard coded latest version.
db := NewTestDBWithVersion(t, LatestMigrationVersion)

// We'll now attempt to execute migrations, targeting the latest
// version. But we'll have the DB think the latest version is actually
// less than the current version. This simulates downgrading.
err := db.ExecuteMigrations(TargetLatest, WithLatestVersion(1))
require.ErrorIs(t, err, ErrMigrationDowngrade)
}
10 changes: 9 additions & 1 deletion tapdb/postgres.go
Original file line number Diff line number Diff line change
@@ -141,7 +141,14 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) {

// ExecuteMigrations runs migrations for the Postgres database, depending on the
// target given, either all migrations or up to a given version.
func (s *PostgresStore) ExecuteMigrations(target MigrationTarget) error {
func (s *PostgresStore) ExecuteMigrations(target MigrationTarget,
optFuncs ...MigrateOpt) error {

opts := defaultMigrateOptions()
for _, optFunc := range optFuncs {
optFunc(opts)
}

driver, err := postgres_migrate.WithInstance(
s.DB, &postgres_migrate.Config{},
)
@@ -152,6 +159,7 @@ func (s *PostgresStore) ExecuteMigrations(target MigrationTarget) error {
postgresFS := newReplacerFS(sqlSchemas, postgresSchemaReplacements)
return applyMigrations(
postgresFS, driver, "sqlc/migrations", s.cfg.DBName, target,
opts,
)
}

11 changes: 9 additions & 2 deletions tapdb/sqlite.go
Original file line number Diff line number Diff line change
@@ -151,7 +151,14 @@ func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) {

// ExecuteMigrations runs migrations for the sqlite database, depending on the
// target given, either all migrations or up to a given version.
func (s *SqliteStore) ExecuteMigrations(target MigrationTarget) error {
func (s *SqliteStore) ExecuteMigrations(target MigrationTarget,
optFuncs ...MigrateOpt) error {

opts := defaultMigrateOptions()
for _, optFunc := range optFuncs {
optFunc(opts)
}

driver, err := sqlite_migrate.WithInstance(
s.DB, &sqlite_migrate.Config{},
)
@@ -161,7 +168,7 @@ func (s *SqliteStore) ExecuteMigrations(target MigrationTarget) error {

sqliteFS := newReplacerFS(sqlSchemas, sqliteSchemaReplacements)
return applyMigrations(
sqliteFS, driver, "sqlc/migrations", "sqlite", target,
sqliteFS, driver, "sqlc/migrations", "sqlite", target, opts,
)
}