From 4e4121335ae4cf5d6628e6fc04930e080596b49e Mon Sep 17 00:00:00 2001 From: jholdstock Date: Mon, 24 Jun 2024 09:53:45 +0100 Subject: [PATCH] database: Store xpub keys in a bucket. **Warning: This commit contains a database upgrade.** In order to add future support for retiring xpub keys, the database is upgraded such that the keys are now stored in a dedicated bucket which can hold multiple values rather than storing a single key as individual values in the root bucket. A new ID field is added to distinguish between keys. This ID is added to every ticket record in the database in order to track which pubkey was used for each ticket. A new field named "Retired" has also been added to pubkeys. It is a unix timestamp representing the moment the key was retired, or zero for the currently active key. --- database/database.go | 15 ++--- database/feexpub.go | 83 ++++++++++++++++++-------- database/feexpub_test.go | 29 +++++++-- database/ticket.go | 6 ++ database/ticket_test.go | 2 + database/upgrade_v5.go | 92 +++++++++++++++++++++++++++++ database/upgrades.go | 12 +++- internal/webapi/addressgenerator.go | 6 ++ internal/webapi/getfeeaddress.go | 1 + 9 files changed, 208 insertions(+), 38 deletions(-) create mode 100644 database/upgrade_v5.go diff --git a/database/database.go b/database/database.go index 11d3cfad..f7409ad7 100644 --- a/database/database.go +++ b/database/database.go @@ -39,14 +39,13 @@ var ( voteChangeBktK = []byte("votechangebkt") // version is the current database version. versionK = []byte("version") - // feeXPub is the extended public key used for collecting VSP fees. - feeXPubK = []byte("feeXPub") + // xPubBktK stores current and historic extended public keys used for + // collecting VSP fees. + xPubBktK = []byte("xpubbkt") // cookieSecret is the secret key for initializing the cookie store. cookieSecretK = []byte("cookieSecret") // privatekey is the private key. privateKeyK = []byte("privatekey") - // lastaddressindex is the index of the last address used for fees. - lastAddressIndexK = []byte("lastaddressindex") // altSignAddrBktK stores alternate signing addresses. altSignAddrBktK = []byte("altsigbkt") ) @@ -137,12 +136,14 @@ func CreateNew(dbFile, feeXPub string) error { return err } - // Store fee xpub. - xpub := FeeXPub{ + // Insert the initial fee xpub with ID 0. + newKey := FeeXPub{ + ID: 0, Key: feeXPub, LastUsedIdx: 0, + Retired: 0, } - err = insertFeeXPub(tx, xpub) + err = insertFeeXPub(tx, newKey) if err != nil { return err } diff --git a/database/feexpub.go b/database/feexpub.go index df3ab7a6..201924c0 100644 --- a/database/feexpub.go +++ b/database/feexpub.go @@ -5,14 +5,20 @@ package database import ( + "encoding/json" "fmt" bolt "go.etcd.io/bbolt" ) +// FeeXPub is serialized to json and stored in bbolt db. type FeeXPub struct { - Key string - LastUsedIdx uint32 + ID uint32 `json:"id"` + Key string `json:"key"` + LastUsedIdx uint32 `json:"lastusedidx"` + // Retired is a unix timestamp representing the moment the key was retired, + // or zero for the currently active key. + Retired int64 `json:"retired"` } // insertFeeXPub stores the provided pubkey in the database, regardless of @@ -20,43 +26,74 @@ type FeeXPub struct { func insertFeeXPub(tx *bolt.Tx, xpub FeeXPub) error { vspBkt := tx.Bucket(vspBktK) - err := vspBkt.Put(feeXPubK, []byte(xpub.Key)) + keyBkt, err := vspBkt.CreateBucketIfNotExists(xPubBktK) if err != nil { - return err + return fmt.Errorf("failed to get %s bucket: %w", string(xPubBktK), err) + } + + keyBytes, err := json.Marshal(xpub) + if err != nil { + return fmt.Errorf("could not marshal xpub: %w", err) } - return vspBkt.Put(lastAddressIndexK, uint32ToBytes(xpub.LastUsedIdx)) + err = keyBkt.Put(uint32ToBytes(xpub.ID), keyBytes) + if err != nil { + return fmt.Errorf("could not store xpub: %w", err) + } + + return nil } -// FeeXPub retrieves the extended pubkey used for generating fee addresses -// from the database. +// FeeXPub retrieves the currently active extended pubkey used for generating +// fee addresses from the database. func (vdb *VspDatabase) FeeXPub() (FeeXPub, error) { - var feeXPub string - var idx uint32 + xpubs, err := vdb.AllXPubs() + if err != nil { + return FeeXPub{}, err + } + + // Find the active xpub - the one with the highest ID. + var highest uint32 + for id := range xpubs { + if id > highest { + highest = id + } + } + + return xpubs[highest], nil +} + +// AllXPubs retrieves the current and any retired extended pubkeys from the +// database. +func (vdb *VspDatabase) AllXPubs() (map[uint32]FeeXPub, error) { + xpubs := make(map[uint32]FeeXPub) + err := vdb.db.View(func(tx *bolt.Tx) error { - vspBkt := tx.Bucket(vspBktK) + bkt := tx.Bucket(vspBktK).Bucket(xPubBktK) - // Get the key. - xpubBytes := vspBkt.Get(feeXPubK) - if xpubBytes == nil { - return nil + if bkt == nil { + return fmt.Errorf("%s bucket doesn't exist", string(xPubBktK)) } - feeXPub = string(xpubBytes) - // Get the last used address index. - idxBytes := vspBkt.Get(lastAddressIndexK) - if idxBytes == nil { + err := bkt.ForEach(func(k, v []byte) error { + var xpub FeeXPub + err := json.Unmarshal(v, &xpub) + if err != nil { + return fmt.Errorf("could not unmarshal xpub key: %w", err) + } + + xpubs[bytesToUint32(k)] = xpub + return nil + }) + if err != nil { + return fmt.Errorf("error iterating over %s bucket: %w", string(xPubBktK), err) } - idx = bytesToUint32(idxBytes) return nil }) - if err != nil { - return FeeXPub{}, fmt.Errorf("could not retrieve fee xpub: %w", err) - } - return FeeXPub{Key: feeXPub, LastUsedIdx: idx}, nil + return xpubs, err } // SetLastAddressIndex updates the last index used to derive a new fee address diff --git a/database/feexpub_test.go b/database/feexpub_test.go index 26804607..3d470de1 100644 --- a/database/feexpub_test.go +++ b/database/feexpub_test.go @@ -9,8 +9,7 @@ import ( ) func testFeeXPub(t *testing.T) { - // A newly created DB should store the fee xpub it was initialized with, and - // the last used index should be 0. + // A newly created DB should store the fee xpub it was initialized with. retrievedXPub, err := db.FeeXPub() if err != nil { t.Fatalf("error getting fee xpub: %v", err) @@ -20,8 +19,15 @@ func testFeeXPub(t *testing.T) { t.Fatalf("expected fee xpub %v, got %v", feeXPub, retrievedXPub.Key) } + // The ID, last used index and retirement timestamp should all be 0 + if retrievedXPub.ID != 0 { + t.Fatalf("expected xpub ID 0, got %d", retrievedXPub.ID) + } if retrievedXPub.LastUsedIdx != 0 { - t.Fatalf("retrieved addr index value didnt match expected") + t.Fatalf("expected xpub last used 0, got %d", retrievedXPub.LastUsedIdx) + } + if retrievedXPub.Retired != 0 { + t.Fatalf("expected xpub retirement 0, got %d", retrievedXPub.Retired) } // Update address index. @@ -34,9 +40,20 @@ func testFeeXPub(t *testing.T) { // Check for updated value. retrievedXPub, err = db.FeeXPub() if err != nil { - t.Fatalf("error getting address index: %v", err) + t.Fatalf("error getting fee xpub: %v", err) + } + if retrievedXPub.LastUsedIdx != idx { + t.Fatalf("expected xpub last used %d, got %d", idx, retrievedXPub.LastUsedIdx) + } + + // Key, ID and retirement timestamp should be unchanged. + if retrievedXPub.Key != feeXPub { + t.Fatalf("expected fee xpub %v, got %v", feeXPub, retrievedXPub.Key) + } + if retrievedXPub.ID != 0 { + t.Fatalf("expected xpub ID 0, got %d", retrievedXPub.ID) } - if idx != retrievedXPub.LastUsedIdx { - t.Fatalf("retrieved addr index value didnt match expected") + if retrievedXPub.Retired != 0 { + t.Fatalf("expected xpub retirement 0, got %d", retrievedXPub.Retired) } } diff --git a/database/ticket.go b/database/ticket.go index 38a444fe..d596d10b 100644 --- a/database/ticket.go +++ b/database/ticket.go @@ -50,6 +50,7 @@ var ( hashK = []byte("Hash") purchaseHeightK = []byte("PurchaseHeight") commitmentAddressK = []byte("CommitmentAddress") + feeAddressXPubIDK = []byte("FeeAddressXPubID") feeAddressIndexK = []byte("FeeAddressIndex") feeAddressK = []byte("FeeAddress") feeAmountK = []byte("FeeAmount") @@ -69,6 +70,7 @@ type Ticket struct { Hash string PurchaseHeight int64 CommitmentAddress string + FeeAddressXPubID uint32 FeeAddressIndex uint32 FeeAddress string FeeAmount int64 @@ -184,6 +186,9 @@ func putTicketInBucket(bkt *bolt.Bucket, ticket Ticket) error { if err = bkt.Put(feeAddressIndexK, uint32ToBytes(ticket.FeeAddressIndex)); err != nil { return err } + if err = bkt.Put(feeAddressXPubIDK, uint32ToBytes(ticket.FeeAddressXPubID)); err != nil { + return err + } if err = bkt.Put(feeAmountK, int64ToBytes(ticket.FeeAmount)); err != nil { return err } @@ -216,6 +221,7 @@ func getTicketFromBkt(bkt *bolt.Bucket) (Ticket, error) { ticket.Outcome = TicketOutcome(bkt.Get(outcomeK)) ticket.PurchaseHeight = bytesToInt64(bkt.Get(purchaseHeightK)) + ticket.FeeAddressXPubID = bytesToUint32(bkt.Get(feeAddressXPubIDK)) ticket.FeeAddressIndex = bytesToUint32(bkt.Get(feeAddressIndexK)) ticket.FeeAmount = bytesToInt64(bkt.Get(feeAmountK)) ticket.FeeExpiration = bytesToInt64(bkt.Get(feeExpirationK)) diff --git a/database/ticket_test.go b/database/ticket_test.go index 812b4b54..e3a89a1d 100644 --- a/database/ticket_test.go +++ b/database/ticket_test.go @@ -17,6 +17,7 @@ func exampleTicket() Ticket { Hash: randString(64, hexCharset), CommitmentAddress: randString(35, addrCharset), FeeAddressIndex: 12345, + FeeAddressXPubID: 10, FeeAddress: randString(35, addrCharset), FeeAmount: 10000000, FeeExpiration: 4, @@ -129,6 +130,7 @@ func testUpdateTicket(t *testing.T) { ticket.FeeAmount = ticket.FeeAmount + 1 ticket.FeeExpiration = ticket.FeeExpiration + 1 ticket.VoteChoices = map[string]string{"New agenda": "New value"} + ticket.FeeAddressXPubID = 20 err = db.UpdateTicket(ticket) if err != nil { diff --git a/database/upgrade_v5.go b/database/upgrade_v5.go new file mode 100644 index 00000000..f09a3c29 --- /dev/null +++ b/database/upgrade_v5.go @@ -0,0 +1,92 @@ +// Copyright (c) 2024 The Decred developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package database + +import ( + "errors" + "fmt" + + "github.com/decred/slog" + bolt "go.etcd.io/bbolt" +) + +func xPubBucketUpgrade(db *bolt.DB, log slog.Logger) error { + log.Infof("Upgrading database to version %d", xPubBucketVersion) + + // feeXPub is the key which was used prior to this upgrade to store the xpub + // in the root bucket. + feeXPubK := []byte("feeXPub") + + // lastaddressindex is the key which was used prior to this upgrade to store + // the index of the last used address in the root bucket. + lastAddressIndexK := []byte("lastaddressindex") + + // Run the upgrade in a single database transaction so it can be safely + // rolled back if an error is encountered. + err := db.Update(func(tx *bolt.Tx) error { + vspBkt := tx.Bucket(vspBktK) + ticketBkt := vspBkt.Bucket(ticketBktK) + + // Retrieve the current xpub. + xpubBytes := vspBkt.Get(feeXPubK) + if xpubBytes == nil { + return errors.New("xpub not found") + } + feeXPub := string(xpubBytes) + + // Retrieve the current last addr index. Could be nil if this xpub was + // never used. + idxBytes := vspBkt.Get(lastAddressIndexK) + var idx uint32 + if idxBytes != nil { + idx = bytesToUint32(idxBytes) + } + + // Delete the old values from the database. + err := vspBkt.Delete(feeXPubK) + if err != nil { + return fmt.Errorf("could not delete xpub: %w", err) + } + err = vspBkt.Delete(lastAddressIndexK) + if err != nil { + return fmt.Errorf("could not delete last addr idx: %w", err) + } + + // Insert the key into the bucket. + newXpub := FeeXPub{ + ID: 0, + Key: feeXPub, + LastUsedIdx: idx, + Retired: 0, + } + err = insertFeeXPub(tx, newXpub) + if err != nil { + return fmt.Errorf("failed to store xpub in new bucket: %w", err) + } + + // Update all existing tickets with xpub key ID 0. + err = ticketBkt.ForEachBucket(func(k []byte) error { + return ticketBkt.Bucket(k).Put(feeAddressXPubIDK, uint32ToBytes(0)) + }) + if err != nil { + return fmt.Errorf("setting ticket xpub ID to 0 failed: %w", err) + } + + // Update database version. + err = vspBkt.Put(versionK, uint32ToBytes(xPubBucketVersion)) + if err != nil { + return fmt.Errorf("failed to update db version: %w", err) + } + + return nil + }) + if err != nil { + return err + } + + log.Info("Upgrade completed") + + return nil +} diff --git a/database/upgrades.go b/database/upgrades.go index 946abe80..74994a84 100644 --- a/database/upgrades.go +++ b/database/upgrades.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2022 The Decred developers +// Copyright (c) 2021-2024 The Decred developers // Use of this source code is governed by an ISC // license that can be found in the LICENSE file. @@ -30,10 +30,17 @@ const ( // to verify messages sent to the vspd. altSignAddrVersion = 4 + // xPubBucketVersion changes how the xpub key and its associated addr index + // are stored. Previously only a single key was supported because it was + // stored as a single value in the root bucket. Now a dedicated bucket which + // can hold multiple keys is used, enabling support for historic retired + // keys as well as the current key. + xPubBucketVersion = 5 + // latestVersion is the latest version of the database that is understood by // vspd. Databases with recorded versions higher than this will fail to open // (meaning any upgrades prevent reverting to older software). - latestVersion = altSignAddrVersion + latestVersion = xPubBucketVersion ) // upgrades maps between old database versions and the upgrade function to @@ -42,6 +49,7 @@ var upgrades = []func(tx *bolt.DB, log slog.Logger) error{ initialVersion: removeOldFeeTxUpgrade, removeOldFeeTxVersion: ticketBucketUpgrade, ticketBucketVersion: altSignAddrUpgrade, + altSignAddrVersion: xPubBucketUpgrade, } // v1Ticket has the json tags required to unmarshal tickets stored in the diff --git a/internal/webapi/addressgenerator.go b/internal/webapi/addressgenerator.go index 73d7c7af..0b60ca6d 100644 --- a/internal/webapi/addressgenerator.go +++ b/internal/webapi/addressgenerator.go @@ -18,6 +18,7 @@ type addressGenerator struct { external *hdkeychain.ExtendedKey netParams *chaincfg.Params lastUsedIndex uint32 + feeXPubID uint32 log slog.Logger } @@ -41,10 +42,15 @@ func newAddressGenerator(xPub database.FeeXPub, netParams *chaincfg.Params, log external: external, netParams: netParams, lastUsedIndex: xPub.LastUsedIdx, + feeXPubID: xPub.ID, log: log, }, nil } +func (m *addressGenerator) xPubID() uint32 { + return m.feeXPubID +} + // nextAddress increments the last used address counter and returns a new // address. It will skip any address index which causes an ErrInvalidChild. // Not safe for concurrent access. diff --git a/internal/webapi/getfeeaddress.go b/internal/webapi/getfeeaddress.go index 52404a8f..c2dec86b 100644 --- a/internal/webapi/getfeeaddress.go +++ b/internal/webapi/getfeeaddress.go @@ -195,6 +195,7 @@ func (w *WebAPI) feeAddress(c *gin.Context) { PurchaseHeight: purchaseHeight, CommitmentAddress: commitmentAddress, FeeAddressIndex: newAddressIdx, + FeeAddressXPubID: w.addrGen.xPubID(), FeeAddress: newAddress, Confirmed: confirmed, FeeAmount: int64(fee),