Skip to content

Commit

Permalink
database: Store xpub keys in a bucket.
Browse files Browse the repository at this point in the history
**Warning: This commit contains a database upgrade.**

In order to add future support for retiring xpub keys, the database is
upgraded such that the keys are now stored in a dedicated bucket which
can hold multiple values rather than storing a single key as individual
values in the root bucket.

A new ID field is added to distinguish between keys. This ID is added to
every ticket record in the database in order to track which pubkey was
used for each ticket.

A new field named "Retired" has also been added to pubkeys. It is a unix
timestamp representing the moment the key was retired, or zero for the
currently active key.
  • Loading branch information
jholdstock committed Jun 27, 2024
1 parent 1fa81d3 commit 4e41213
Show file tree
Hide file tree
Showing 9 changed files with 208 additions and 38 deletions.
15 changes: 8 additions & 7 deletions database/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,13 @@ var (
voteChangeBktK = []byte("votechangebkt")
// version is the current database version.
versionK = []byte("version")
// feeXPub is the extended public key used for collecting VSP fees.
feeXPubK = []byte("feeXPub")
// xPubBktK stores current and historic extended public keys used for
// collecting VSP fees.
xPubBktK = []byte("xpubbkt")
// cookieSecret is the secret key for initializing the cookie store.
cookieSecretK = []byte("cookieSecret")
// privatekey is the private key.
privateKeyK = []byte("privatekey")
// lastaddressindex is the index of the last address used for fees.
lastAddressIndexK = []byte("lastaddressindex")
// altSignAddrBktK stores alternate signing addresses.
altSignAddrBktK = []byte("altsigbkt")
)
Expand Down Expand Up @@ -137,12 +136,14 @@ func CreateNew(dbFile, feeXPub string) error {
return err
}

// Store fee xpub.
xpub := FeeXPub{
// Insert the initial fee xpub with ID 0.
newKey := FeeXPub{
ID: 0,
Key: feeXPub,
LastUsedIdx: 0,
Retired: 0,
}
err = insertFeeXPub(tx, xpub)
err = insertFeeXPub(tx, newKey)
if err != nil {
return err
}
Expand Down
83 changes: 60 additions & 23 deletions database/feexpub.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,58 +5,95 @@
package database

import (
"encoding/json"
"fmt"

bolt "go.etcd.io/bbolt"
)

// FeeXPub is serialized to json and stored in bbolt db.
type FeeXPub struct {
Key string
LastUsedIdx uint32
ID uint32 `json:"id"`
Key string `json:"key"`
LastUsedIdx uint32 `json:"lastusedidx"`
// Retired is a unix timestamp representing the moment the key was retired,
// or zero for the currently active key.
Retired int64 `json:"retired"`
}

// insertFeeXPub stores the provided pubkey in the database, regardless of
// whether a value pre-exists.
func insertFeeXPub(tx *bolt.Tx, xpub FeeXPub) error {
vspBkt := tx.Bucket(vspBktK)

err := vspBkt.Put(feeXPubK, []byte(xpub.Key))
keyBkt, err := vspBkt.CreateBucketIfNotExists(xPubBktK)
if err != nil {
return err
return fmt.Errorf("failed to get %s bucket: %w", string(xPubBktK), err)
}

keyBytes, err := json.Marshal(xpub)
if err != nil {
return fmt.Errorf("could not marshal xpub: %w", err)
}

return vspBkt.Put(lastAddressIndexK, uint32ToBytes(xpub.LastUsedIdx))
err = keyBkt.Put(uint32ToBytes(xpub.ID), keyBytes)
if err != nil {
return fmt.Errorf("could not store xpub: %w", err)
}

return nil
}

// FeeXPub retrieves the extended pubkey used for generating fee addresses
// from the database.
// FeeXPub retrieves the currently active extended pubkey used for generating
// fee addresses from the database.
func (vdb *VspDatabase) FeeXPub() (FeeXPub, error) {
var feeXPub string
var idx uint32
xpubs, err := vdb.AllXPubs()
if err != nil {
return FeeXPub{}, err
}

// Find the active xpub - the one with the highest ID.
var highest uint32
for id := range xpubs {
if id > highest {
highest = id
}
}

return xpubs[highest], nil
}

// AllXPubs retrieves the current and any retired extended pubkeys from the
// database.
func (vdb *VspDatabase) AllXPubs() (map[uint32]FeeXPub, error) {
xpubs := make(map[uint32]FeeXPub)

err := vdb.db.View(func(tx *bolt.Tx) error {
vspBkt := tx.Bucket(vspBktK)
bkt := tx.Bucket(vspBktK).Bucket(xPubBktK)

// Get the key.
xpubBytes := vspBkt.Get(feeXPubK)
if xpubBytes == nil {
return nil
if bkt == nil {
return fmt.Errorf("%s bucket doesn't exist", string(xPubBktK))
}
feeXPub = string(xpubBytes)

// Get the last used address index.
idxBytes := vspBkt.Get(lastAddressIndexK)
if idxBytes == nil {
err := bkt.ForEach(func(k, v []byte) error {
var xpub FeeXPub
err := json.Unmarshal(v, &xpub)
if err != nil {
return fmt.Errorf("could not unmarshal xpub key: %w", err)
}

xpubs[bytesToUint32(k)] = xpub

return nil
})
if err != nil {
return fmt.Errorf("error iterating over %s bucket: %w", string(xPubBktK), err)
}
idx = bytesToUint32(idxBytes)

return nil
})
if err != nil {
return FeeXPub{}, fmt.Errorf("could not retrieve fee xpub: %w", err)
}

return FeeXPub{Key: feeXPub, LastUsedIdx: idx}, nil
return xpubs, err
}

// SetLastAddressIndex updates the last index used to derive a new fee address
Expand Down
29 changes: 23 additions & 6 deletions database/feexpub_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@ import (
)

func testFeeXPub(t *testing.T) {
// A newly created DB should store the fee xpub it was initialized with, and
// the last used index should be 0.
// A newly created DB should store the fee xpub it was initialized with.
retrievedXPub, err := db.FeeXPub()
if err != nil {
t.Fatalf("error getting fee xpub: %v", err)
Expand All @@ -20,8 +19,15 @@ func testFeeXPub(t *testing.T) {
t.Fatalf("expected fee xpub %v, got %v", feeXPub, retrievedXPub.Key)
}

// The ID, last used index and retirement timestamp should all be 0
if retrievedXPub.ID != 0 {
t.Fatalf("expected xpub ID 0, got %d", retrievedXPub.ID)
}
if retrievedXPub.LastUsedIdx != 0 {
t.Fatalf("retrieved addr index value didnt match expected")
t.Fatalf("expected xpub last used 0, got %d", retrievedXPub.LastUsedIdx)
}
if retrievedXPub.Retired != 0 {
t.Fatalf("expected xpub retirement 0, got %d", retrievedXPub.Retired)
}

// Update address index.
Expand All @@ -34,9 +40,20 @@ func testFeeXPub(t *testing.T) {
// Check for updated value.
retrievedXPub, err = db.FeeXPub()
if err != nil {
t.Fatalf("error getting address index: %v", err)
t.Fatalf("error getting fee xpub: %v", err)
}
if retrievedXPub.LastUsedIdx != idx {
t.Fatalf("expected xpub last used %d, got %d", idx, retrievedXPub.LastUsedIdx)
}

// Key, ID and retirement timestamp should be unchanged.
if retrievedXPub.Key != feeXPub {
t.Fatalf("expected fee xpub %v, got %v", feeXPub, retrievedXPub.Key)
}
if retrievedXPub.ID != 0 {
t.Fatalf("expected xpub ID 0, got %d", retrievedXPub.ID)
}
if idx != retrievedXPub.LastUsedIdx {
t.Fatalf("retrieved addr index value didnt match expected")
if retrievedXPub.Retired != 0 {
t.Fatalf("expected xpub retirement 0, got %d", retrievedXPub.Retired)
}
}
6 changes: 6 additions & 0 deletions database/ticket.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ var (
hashK = []byte("Hash")
purchaseHeightK = []byte("PurchaseHeight")
commitmentAddressK = []byte("CommitmentAddress")
feeAddressXPubIDK = []byte("FeeAddressXPubID")
feeAddressIndexK = []byte("FeeAddressIndex")
feeAddressK = []byte("FeeAddress")
feeAmountK = []byte("FeeAmount")
Expand All @@ -69,6 +70,7 @@ type Ticket struct {
Hash string
PurchaseHeight int64
CommitmentAddress string
FeeAddressXPubID uint32
FeeAddressIndex uint32
FeeAddress string
FeeAmount int64
Expand Down Expand Up @@ -184,6 +186,9 @@ func putTicketInBucket(bkt *bolt.Bucket, ticket Ticket) error {
if err = bkt.Put(feeAddressIndexK, uint32ToBytes(ticket.FeeAddressIndex)); err != nil {
return err
}
if err = bkt.Put(feeAddressXPubIDK, uint32ToBytes(ticket.FeeAddressXPubID)); err != nil {
return err
}
if err = bkt.Put(feeAmountK, int64ToBytes(ticket.FeeAmount)); err != nil {
return err
}
Expand Down Expand Up @@ -216,6 +221,7 @@ func getTicketFromBkt(bkt *bolt.Bucket) (Ticket, error) {
ticket.Outcome = TicketOutcome(bkt.Get(outcomeK))

ticket.PurchaseHeight = bytesToInt64(bkt.Get(purchaseHeightK))
ticket.FeeAddressXPubID = bytesToUint32(bkt.Get(feeAddressXPubIDK))
ticket.FeeAddressIndex = bytesToUint32(bkt.Get(feeAddressIndexK))
ticket.FeeAmount = bytesToInt64(bkt.Get(feeAmountK))
ticket.FeeExpiration = bytesToInt64(bkt.Get(feeExpirationK))
Expand Down
2 changes: 2 additions & 0 deletions database/ticket_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ func exampleTicket() Ticket {
Hash: randString(64, hexCharset),
CommitmentAddress: randString(35, addrCharset),
FeeAddressIndex: 12345,
FeeAddressXPubID: 10,
FeeAddress: randString(35, addrCharset),
FeeAmount: 10000000,
FeeExpiration: 4,
Expand Down Expand Up @@ -129,6 +130,7 @@ func testUpdateTicket(t *testing.T) {
ticket.FeeAmount = ticket.FeeAmount + 1
ticket.FeeExpiration = ticket.FeeExpiration + 1
ticket.VoteChoices = map[string]string{"New agenda": "New value"}
ticket.FeeAddressXPubID = 20

err = db.UpdateTicket(ticket)
if err != nil {
Expand Down
92 changes: 92 additions & 0 deletions database/upgrade_v5.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// Copyright (c) 2024 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package database

import (
"errors"
"fmt"

"github.com/decred/slog"
bolt "go.etcd.io/bbolt"
)

func xPubBucketUpgrade(db *bolt.DB, log slog.Logger) error {
log.Infof("Upgrading database to version %d", xPubBucketVersion)

// feeXPub is the key which was used prior to this upgrade to store the xpub
// in the root bucket.
feeXPubK := []byte("feeXPub")

// lastaddressindex is the key which was used prior to this upgrade to store
// the index of the last used address in the root bucket.
lastAddressIndexK := []byte("lastaddressindex")

// Run the upgrade in a single database transaction so it can be safely
// rolled back if an error is encountered.
err := db.Update(func(tx *bolt.Tx) error {
vspBkt := tx.Bucket(vspBktK)
ticketBkt := vspBkt.Bucket(ticketBktK)

// Retrieve the current xpub.
xpubBytes := vspBkt.Get(feeXPubK)
if xpubBytes == nil {
return errors.New("xpub not found")
}
feeXPub := string(xpubBytes)

// Retrieve the current last addr index. Could be nil if this xpub was
// never used.
idxBytes := vspBkt.Get(lastAddressIndexK)
var idx uint32
if idxBytes != nil {
idx = bytesToUint32(idxBytes)
}

// Delete the old values from the database.
err := vspBkt.Delete(feeXPubK)
if err != nil {
return fmt.Errorf("could not delete xpub: %w", err)
}
err = vspBkt.Delete(lastAddressIndexK)
if err != nil {
return fmt.Errorf("could not delete last addr idx: %w", err)
}

// Insert the key into the bucket.
newXpub := FeeXPub{
ID: 0,
Key: feeXPub,
LastUsedIdx: idx,
Retired: 0,
}
err = insertFeeXPub(tx, newXpub)
if err != nil {
return fmt.Errorf("failed to store xpub in new bucket: %w", err)
}

// Update all existing tickets with xpub key ID 0.
err = ticketBkt.ForEachBucket(func(k []byte) error {
return ticketBkt.Bucket(k).Put(feeAddressXPubIDK, uint32ToBytes(0))
})
if err != nil {
return fmt.Errorf("setting ticket xpub ID to 0 failed: %w", err)
}

// Update database version.
err = vspBkt.Put(versionK, uint32ToBytes(xPubBucketVersion))
if err != nil {
return fmt.Errorf("failed to update db version: %w", err)
}

return nil
})
if err != nil {
return err
}

log.Info("Upgrade completed")

return nil
}
12 changes: 10 additions & 2 deletions database/upgrades.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) 2021-2022 The Decred developers
// Copyright (c) 2021-2024 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

Expand Down Expand Up @@ -30,10 +30,17 @@ const (
// to verify messages sent to the vspd.
altSignAddrVersion = 4

// xPubBucketVersion changes how the xpub key and its associated addr index
// are stored. Previously only a single key was supported because it was
// stored as a single value in the root bucket. Now a dedicated bucket which
// can hold multiple keys is used, enabling support for historic retired
// keys as well as the current key.
xPubBucketVersion = 5

// latestVersion is the latest version of the database that is understood by
// vspd. Databases with recorded versions higher than this will fail to open
// (meaning any upgrades prevent reverting to older software).
latestVersion = altSignAddrVersion
latestVersion = xPubBucketVersion
)

// upgrades maps between old database versions and the upgrade function to
Expand All @@ -42,6 +49,7 @@ var upgrades = []func(tx *bolt.DB, log slog.Logger) error{
initialVersion: removeOldFeeTxUpgrade,
removeOldFeeTxVersion: ticketBucketUpgrade,
ticketBucketVersion: altSignAddrUpgrade,
altSignAddrVersion: xPubBucketUpgrade,
}

// v1Ticket has the json tags required to unmarshal tickets stored in the
Expand Down
Loading

0 comments on commit 4e41213

Please sign in to comment.