Merge pull request #10338 from ellemouton/graphSQLMigFreeze

graph/db: freeze the SQL migration code

Author: ellemouton

config_builder.go

@@ -36,6 +36,8 @@ import (
 	"github.com/lightningnetwork/lnd/fn/v2"
 	"github.com/lightningnetwork/lnd/funding"
 	graphdb "github.com/lightningnetwork/lnd/graph/db"
+	graphdbmig1 "github.com/lightningnetwork/lnd/graph/db/migration1"
+	graphmig1sqlc "github.com/lightningnetwork/lnd/graph/db/migration1/sqlc"
 	"github.com/lightningnetwork/lnd/htlcswitch"
 	"github.com/lightningnetwork/lnd/invoices"
 	"github.com/lightningnetwork/lnd/keychain"
@@ -1134,13 +1136,14 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 			}
 
 			graphMig := func(tx *sqlc.Queries) error {
-				cfg := &graphdb.SQLStoreConfig{
+				cfg := &graphdbmig1.SQLStoreConfig{
 					//nolint:ll
 					ChainHash: *d.cfg.ActiveNetParams.GenesisHash,
 					QueryCfg:  queryCfg,
 				}
-				err := graphdb.MigrateGraphToSQL(
-					ctx, cfg, dbs.ChanStateDB.Backend, tx,
+				err := graphdbmig1.MigrateGraphToSQL(
+					ctx, cfg, dbs.ChanStateDB.Backend,
+					graphmig1sqlc.New(tx.GetTx()),
 				)
 				if err != nil {
 					return fmt.Errorf("failed to migrate "+

docs/release-notes/release-notes-0.21.0.md

@@ -54,8 +54,14 @@
 
 ## Database
 
+* Freeze the [graph SQL migration 
+  code](https://github.com/lightningnetwork/lnd/pull/10338) to prevent the 
+  need for maintenance as the sqlc code evolves. 
+
 ## Code Health
 
 ## Tooling and Documentation
 
 # Contributors (Alphabetical Order)
+
+* Elle Mouton

go.mod

@@ -202,6 +202,9 @@ require (
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )
 
+// TODO(elle): remove once the gossip V2 sqldb changes have been made.
+replace github.com/lightningnetwork/lnd/sqldb => ./sqldb
+
 // This replace is for https://github.com/advisories/GHSA-25xm-hr59-7c27
 replace github.com/ulikunitz/xz => github.com/ulikunitz/xz v0.5.11
 

go.sum

@@ -382,8 +382,6 @@ github.com/lightningnetwork/lnd/kvdb v1.4.16 h1:9BZgWdDfjmHRHLS97cz39bVuBAqMc4/p
 github.com/lightningnetwork/lnd/kvdb v1.4.16/go.mod h1:HW+bvwkxNaopkz3oIgBV6NEnV4jCEZCACFUcNg4xSjM=
 github.com/lightningnetwork/lnd/queue v1.1.1 h1:99ovBlpM9B0FRCGYJo6RSFDlt8/vOkQQZznVb18iNMI=
 github.com/lightningnetwork/lnd/queue v1.1.1/go.mod h1:7A6nC1Qrm32FHuhx/mi1cieAiBZo5O6l8IBIoQxvkz4=
-github.com/lightningnetwork/lnd/sqldb v1.0.11 h1:X8J3OvdIhJVniQG78Qsp3niErl1zdGMTPvzgiLMWOOo=
-github.com/lightningnetwork/lnd/sqldb v1.0.11/go.mod h1:oOdZ7vjmAUmI9He+aFHTunnxKVefHZAfJttZdz16hSg=
 github.com/lightningnetwork/lnd/ticker v1.1.1 h1:J/b6N2hibFtC7JLV77ULQp++QLtCwT6ijJlbdiZFbSM=
 github.com/lightningnetwork/lnd/ticker v1.1.1/go.mod h1:waPTRAAcwtu7Ji3+3k+u/xH5GHovTsCoSVpho0KDvdA=
 github.com/lightningnetwork/lnd/tlv v1.3.2 h1:MO4FCk7F4k5xPMqVZF6Nb/kOpxlwPrUQpYjmyKny5s0=

graph/db/benchmark_test.go

@@ -6,14 +6,12 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"os"
 	"path"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/btcsuite/btcd/chaincfg"
-	"github.com/btcsuite/btclog/v2"
 	"github.com/lightningnetwork/lnd/batch"
 	"github.com/lightningnetwork/lnd/fn/v2"
 	"github.com/lightningnetwork/lnd/graph/db/models"
@@ -430,137 +428,6 @@ func TestPopulateDBs(t *testing.T) {
 	}
 }
 
-// TestPopulateViaMigration is a helper test that can be used to populate a
-// local native SQL graph from a kvdbgraph using the migration logic.
-//
-// NOTE: the testPostgres variable can be set to true to test with a
-// postgres backend instead of the kvdb-sqlite backend.
-//
-// NOTE: you will need to set the following build tags in order to run this
-// test:
-//
-//	test_native_sql
-//	kvdb_sqlite // If your source is kvdb-sqlite
-//	kvdb_postgres // If your source is kvdb-postgres
-//
-// NOTE: this is a helper test and is not run by default.
-func TestPopulateViaMigration(t *testing.T) {
-	// ======= STEP 0 ===========
-	// Comment out this SKipf line.
-	t.Skipf("Skipping local helper test")
-
-	const (
-		srcBBolt    = "kvdb-bbolt"
-		srcSQLite   = "kvdb-sqlite"
-		srcPostgres = "kvdb-postgres"
-	)
-
-	// ======= STEP 1 ===========
-	// Set your chosen SOURCE type by uncommenting the corresponding line
-	// below. By default, a kvdb-sqlite source is chosen.
-	srcDB := srcSQLite
-	// srcDB := srcBBolt
-	// srcDB := srcPostgres
-
-	// ======= STEP 2 ============
-	// Set this variable to the correct genesis hash of the source
-	// DB. By default, mainnet is assumed.
-	chain := *chaincfg.MainNetParams.GenesisHash
-
-	// ======= STEP 3 (ignore if source is postgres) ==============
-	// If your source destination is bbolt or sqlite, then set this to the
-	// path where your source database can be found.
-	const sourceDBPath = "testdata"
-
-	// ======= STEP 4 (only if source is bbolt!) ============
-	// If your source destination is bbolt, then set this to the name of
-	// the bbolt file that contains the channel graph data.
-	const sourceBBoltName = "channel.db"
-
-	// ======= STEP 5 (only if source is sqlite!) ============
-	// If your source destination is sqlite, then set this to the name of
-	// the sqlite file that contains the channel graph data.
-	const sourceSQLiteName = "channel.sqlite"
-
-	// ======= STEP 6 (only if source is postgres!) ============
-	// Set the DNS of your kvdb postgres instance below. This should be the
-	// same as what you have set in the config of the LND node that
-	// populated the instance (ie, whatever your --db.postgres.dsn is set
-	// to).
-	const kvdbPostgresDNS = "postgres://user@host/db_name"
-
-	// ======== STEP 7 ========================
-	// Finally, pick your destination DB! You can choose either SQLite or
-	// Postgres.
-	testSQLite := true
-
-	// ======== STEP 8 (only if destination is sqlite) ========
-	// Set the path where you want to create the destination SQLite
-	// database. This should be a directory that exists and is writable.
-	const destSQLitePath = "testdata"
-
-	// ======== STEP 9 (only if destination is sqlite) ========
-	// Pick a name for your destination SQLite database file.
-	// NOTE: if you run this test again, delete the previously created
-	// file first.
-	const destSQLiteFile = "lnd-graph-test.sqlite"
-
-	// ======== STEP 10 (only if destination is postgres) ========
-	// NB: this has some additional steps:
-	// 1. First, connect to your destination postgres instance: example:
-	//	 $ psql -U ellemouton -d postgres
-	// 2. Now, create the test database:
-	//	CREATE DATABASE graphtest;
-	// NOTE: if you restart this test for postgres, it helps to first drop
-	// the new database & recreate it.
-	// NOTE: the database name that you use above must be whatever you will
-	// use in the DNS you set below.
-	const sqlPostgresDNS = "postgres://user@host/graphtest"
-
-	// ======= YOUR WORK IS DONE =============
-
-	// Connect to source database.
-	var srcKVDB kvdb.Backend
-	switch srcDB {
-	case srcBBolt:
-		srcKVDB = kvdbBBolt(t, sourceDBPath, sourceBBoltName)
-	case srcSQLite:
-		srcKVDB = kvdbSqlite(t, sourceDBPath, sourceSQLiteName)
-	case srcPostgres:
-		srcKVDB = kvdbPostgres(t, kvdbPostgresDNS)
-	default:
-		t.Fatalf("Unsupported source database backend: %s", srcDB)
-	}
-
-	// Connect to destination database.
-	cfg := sqldb.DefaultSQLiteConfig()
-	dstSQL := sqlSQLite(t, destSQLitePath, destSQLiteFile)
-	if !testSQLite {
-		cfg = sqldb.DefaultPostgresConfig()
-		dstSQL = sqlPostgres(t, sqlPostgresDNS)
-	}
-
-	// Set up a logger so we can see the migration progress.
-	logger := btclog.NewDefaultHandler(os.Stdout)
-	UseLogger(btclog.NewSLogger(logger))
-	log.SetLevel(btclog.LevelDebug)
-
-	// Use the graph migration to populate the SQL graph from the
-	// kvdb graph.
-	ctx := t.Context()
-	err := dstSQL.ExecTx(
-		ctx, sqldb.WriteTxOpt(), func(queries SQLQueries) error {
-			return MigrateGraphToSQL(
-				ctx, &SQLStoreConfig{
-					QueryCfg:  cfg,
-					ChainHash: chain,
-				}, srcKVDB, queries,
-			)
-		}, func() {},
-	)
-	require.NoError(t, err)
-}
-
 // syncGraph synchronizes the source graph with the destination graph by
 // copying all nodes and channels from the source to the destination.
 func syncGraph(t *testing.T, src, dest *ChannelGraph) {