Merge pull request #973 from Roasbeef/prevent-db-downgrade

tapdb: implement basic migration downgrade protection

Author: guggero

.github/workflows/main.yaml

@@ -81,6 +81,16 @@ jobs:
       - name: Run test vector creation check
         run: make test-vector-check
 
+  migration-version-check:
+    name: migration version check
+    runs-on: ubuntu-latest
+    steps:
+      - name: git checkout
+        uses: actions/checkout@v3
+
+      - name: Run migration version check
+        run: make migration-version-check
+
   ########################
   # Compilation check.
   ########################

Makefile

@@ -323,6 +323,10 @@ test-vector-check: gen-deterministic-test-vectors
 	@$(call print, "Checking deterministic test vectors.")
 	if test -n "$$(git status | grep -e ".json")"; then echo "Test vectors not updated"; git status; git diff; exit 1; fi
 
+migration-version-check:
+	@$(call print, "Checking migration version.")
+	./scripts/check-migration-latest-version.sh
+
 clean:
 	@$(call print, "Cleaning source.$(NC)")
 	$(RM) coverage.txt

go.mod

@@ -49,7 +49,7 @@ require (
 	google.golang.org/protobuf v1.33.0
 	gopkg.in/macaroon-bakery.v2 v2.1.0
 	gopkg.in/macaroon.v2 v2.1.0
-	modernc.org/sqlite v1.29.8
+	modernc.org/sqlite v1.30.0
 )
 
 require (
@@ -195,7 +195,7 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
-	modernc.org/libc v1.49.3 // indirect
+	modernc.org/libc v1.50.9 // indirect
 	modernc.org/mathutil v1.6.0 // indirect
 	modernc.org/memory v1.8.0 // indirect
 	modernc.org/strutil v1.2.0 // indirect

go.sum

@@ -525,8 +525,6 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
-github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
@@ -1143,18 +1141,18 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
-modernc.org/cc/v4 v4.20.0 h1:45Or8mQfbUqJOG9WaxvlFYOAQO0lQ5RvqBcFCXngjxk=
-modernc.org/cc/v4 v4.20.0/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
-modernc.org/ccgo/v4 v4.16.0 h1:ofwORa6vx2FMm0916/CkZjpFPSR70VwTjUCe2Eg5BnA=
-modernc.org/ccgo/v4 v4.16.0/go.mod h1:dkNyWIjFrVIZ68DTo36vHK+6/ShBn4ysU61So6PIqCI=
+modernc.org/cc/v4 v4.21.2 h1:dycHFB/jDc3IyacKipCNSDrjIC0Lm1hyoWOZTRR20Lk=
+modernc.org/cc/v4 v4.21.2/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
+modernc.org/ccgo/v4 v4.17.8 h1:yyWBf2ipA0Y9GGz/MmCmi3EFpKgeS7ICrAFes+suEbs=
+modernc.org/ccgo/v4 v4.17.8/go.mod h1:buJnJ6Fn0tyAdP/dqePbrrvLyr6qslFfTbFrCuaYvtA=
 modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
 modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
 modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
 modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
 modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI=
 modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4=
-modernc.org/libc v1.49.3 h1:j2MRCRdwJI2ls/sGbeSk0t2bypOG/uvPZUsGQFDulqg=
-modernc.org/libc v1.49.3/go.mod h1:yMZuGkn7pXbKfoT/M35gFJOAEdSKdxL0q64sF7KqCDo=
+modernc.org/libc v1.50.9 h1:hIWf1uz55lorXQhfoEoezdUHjxzuO6ceshET/yWjSjk=
+modernc.org/libc v1.50.9/go.mod h1:15P6ublJ9FJR8YQCGy8DeQ2Uwur7iW9Hserr/T3OFZE=
 modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
 modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
 modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
@@ -1163,8 +1161,8 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
 modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
 modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
 modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
-modernc.org/sqlite v1.29.8 h1:nGKglNx9K5v0As+zF0/Gcl1kMkmaU1XynYyq92PbsC8=
-modernc.org/sqlite v1.29.8/go.mod h1:lQPm27iqa4UNZpmr4Aor0MH0HkCLbt1huYDfWylLZFk=
+modernc.org/sqlite v1.30.0 h1:8YhPUs/HTnlEgErn/jSYQTwHN/ex8CjHHjg+K9iG7LM=
+modernc.org/sqlite v1.30.0/go.mod h1:cgkTARJ9ugeXSNaLBPK3CqbOe7Ec7ZhWPoMFGldEYEw=
 modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
 modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

scripts/check-migration-latest-version.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+# Get the latest version number from the migration file names.
+migrations_path="tapdb/sqlc/migrations"
+latest_file_version=$(ls -r $migrations_path | grep .up.sql | head -1 | cut -d_ -f1)
+
+# Force base 10 interpretation, getting rid of the leading zeroes.
+latest_file_version=$((10#$latest_file_version))
+
+# Check the value in migrations.go.
+file_path="tapdb/migrations.go"
+latest_code_version=$(grep -oP 'LatestMigrationVersion\s*=\s*\K\d+' "$file_path")
+
+if [ "$latest_file_version" -ne "$latest_code_version" ]; then
+    echo "Latest migration version in file names: $latest_file_version"
+    echo "Latest migration version in code: $latest_code_version"
+    exit 1
+fi
+
+echo "Latest migration version in file names and code match: $latest_file_version"

tapdb/migrations.go

@@ -3,6 +3,7 @@ package tapdb
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"io"
 	"io/fs"
 	"net/http"
@@ -12,6 +13,16 @@ import (
 	"github.com/golang-migrate/migrate/v4"
 	"github.com/golang-migrate/migrate/v4/database"
 	"github.com/golang-migrate/migrate/v4/source/httpfs"
+	"github.com/lightninglabs/taproot-assets/fn"
+)
+
+const (
+	// LatestMigrationVersion is the latest migration version of the
+	// database.  This is used to implement downgrade protection for the
+	// daemon.
+	//
+	// NOTE: This MUST be updated when a new migration is added.
+	LatestMigrationVersion = 21
 )
 
 // MigrationTarget is a functional option that can be passed to applyMigrations
@@ -34,6 +45,36 @@ var (
 	}
 )
 
+var (
+	// ErrMigrationDowngrade is returned when a database downgrade is
+	// detected.
+	ErrMigrationDowngrade = errors.New("database downgrade detected")
+)
+
+// migrationOption is a functional option that can be passed to migrate related
+// methods to modify their behavior.
+type migrateOptions struct {
+	latestVersion fn.Option[uint]
+}
+
+// defaultMigrateOptions returns a new migrateOptions instance with default
+// settings.
+func defaultMigrateOptions() *migrateOptions {
+	return &migrateOptions{}
+}
+
+// MigrateOpt is a functional option that can be passed to migrate related
+// methods to modify behavior.
+type MigrateOpt func(*migrateOptions)
+
+// WithLatestVersion allows callers to override the default latest version
+// setting.
+func WithLatestVersion(version uint) MigrateOpt {
+	return func(o *migrateOptions) {
+		o.latestVersion = fn.Some(version)
+	}
+}
+
 // migrationLogger is a logger that wraps the passed btclog.Logger so it can be
 // used to log migrations.
 type migrationLogger struct {
@@ -72,7 +113,7 @@ func (m *migrationLogger) Verbose() bool {
 // system under the given path, using the passed database driver and database
 // name, up to or down to the given target version.
 func applyMigrations(fs fs.FS, driver database.Driver, path, dbName string,
-	targetVersion MigrationTarget) error {
+	targetVersion MigrationTarget, opts *migrateOptions) error {
 
 	// With the migrate instance open, we'll create a new migration source
 	// using the embedded file system stored in sqlSchemas. The library
@@ -95,6 +136,16 @@ func applyMigrations(fs fs.FS, driver database.Driver, path, dbName string,
 
 	migrationVersion, _, _ := sqlMigrate.Version()
 
+	// As the down migrations may end up *dropping* data, we want to
+	// prevent that without explicit accounting.
+	latestVersion := opts.latestVersion.UnwrapOr(LatestMigrationVersion)
+	if migrationVersion > latestVersion {
+		return fmt.Errorf("%w: database version is newer than the "+
+			"latest migration version, preventing downgrade: "+
+			"db_version=%v, latest_migration_version=%v",
+			ErrMigrationDowngrade, migrationVersion, latestVersion)
+	}
+
 	log.Infof("Applying migrations from version=%v", migrationVersion)
 
 	// Apply our local logger to the migration instance.

tapdb/migrations_test.go

@@ -118,3 +118,15 @@ func TestMigration15(t *testing.T) {
 		t, wire.TxWitness{{0xbb}}, assets[0].PrevWitnesses[1].TxWitness,
 	)
 }
+
+// TestMigrationDowngrade tests that downgrading the database is prevented.
+func TestMigrationDowngrade(t *testing.T) {
+	// For this test, with the current hard coded latest version.
+	db := NewTestDBWithVersion(t, LatestMigrationVersion)
+
+	// We'll now attempt to execute migrations, targeting the latest
+	// version. But we'll have the DB think the latest version is actually
+	// less than the current version. This simulates downgrading.
+	err := db.ExecuteMigrations(TargetLatest, WithLatestVersion(1))
+	require.ErrorIs(t, err, ErrMigrationDowngrade)
+}

tapdb/postgres.go

@@ -141,7 +141,14 @@ func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) {
 
 // ExecuteMigrations runs migrations for the Postgres database, depending on the
 // target given, either all migrations or up to a given version.
-func (s *PostgresStore) ExecuteMigrations(target MigrationTarget) error {
+func (s *PostgresStore) ExecuteMigrations(target MigrationTarget,
+	optFuncs ...MigrateOpt) error {
+
+	opts := defaultMigrateOptions()
+	for _, optFunc := range optFuncs {
+		optFunc(opts)
+	}
+
 	driver, err := postgres_migrate.WithInstance(
 		s.DB, &postgres_migrate.Config{},
 	)
@@ -152,6 +159,7 @@ func (s *PostgresStore) ExecuteMigrations(target MigrationTarget) error {
 	postgresFS := newReplacerFS(sqlSchemas, postgresSchemaReplacements)
 	return applyMigrations(
 		postgresFS, driver, "sqlc/migrations", s.cfg.DBName, target,
+		opts,
 	)
 }
 

tapdb/sqlite.go

@@ -151,7 +151,14 @@ func NewSqliteStore(cfg *SqliteConfig) (*SqliteStore, error) {
 
 // ExecuteMigrations runs migrations for the sqlite database, depending on the
 // target given, either all migrations or up to a given version.
-func (s *SqliteStore) ExecuteMigrations(target MigrationTarget) error {
+func (s *SqliteStore) ExecuteMigrations(target MigrationTarget,
+	optFuncs ...MigrateOpt) error {
+
+	opts := defaultMigrateOptions()
+	for _, optFunc := range optFuncs {
+		optFunc(opts)
+	}
+
 	driver, err := sqlite_migrate.WithInstance(
 		s.DB, &sqlite_migrate.Config{},
 	)
@@ -161,7 +168,7 @@ func (s *SqliteStore) ExecuteMigrations(target MigrationTarget) error {
 
 	sqliteFS := newReplacerFS(sqlSchemas, sqliteSchemaReplacements)
 	return applyMigrations(
-		sqliteFS, driver, "sqlc/migrations", "sqlite", target,
+		sqliteFS, driver, "sqlc/migrations", "sqlite", target, opts,
 	)
 }