Skip to content

Commit cc85bf6

Browse files
fearlessfefearlessfe
authored
Merge pull request #7 from optimism-java/pebble
feat: use pebble db and make storage config unified
2 parents 7a7abe3 + eea88af commit cc85bf6

File tree

10 files changed

+39
-57
lines changed

10 files changed

+39
-57
lines changed

‎beacon/storage.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ type beaconStorageCache struct {
3434

3535
var _ storage2.ContentStorage = &Storage{}
3636

37-
func NewBeaconStorage(config storage2.PortalStorageConfig) (storage2.ContentStorage, error) {
37+
func NewBeaconStorage(config storage2.PortalStorageConfig, db *sql.DB) (storage2.ContentStorage, error) {
3838
bs := &Storage{
3939
storageCapacityInBytes: config.StorageCapacityMB * BytesInMB,
40-
db: config.DB,
40+
db: db,
4141
log: log.New("beacon_storage"),
4242
spec: config.Spec,
4343
cache: &beaconStorageCache{},
@@ -47,7 +47,7 @@ func NewBeaconStorage(config storage2.PortalStorageConfig) (storage2.ContentStor
4747
}
4848

4949
var err error
50-
portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, config.DB)
50+
portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, db)
5151
if err != nil {
5252
return nil, err
5353
}

‎beacon/storage_test.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,10 @@ func genStorage(testDir string) (storage.ContentStorage, error) {
6464
}
6565
config := &storage.PortalStorageConfig{
6666
StorageCapacityMB: 1000,
67-
DB: db,
6867
NodeId: enode.ID(zeroNodeId),
6968
Spec: configs.Mainnet,
7069
}
71-
return NewBeaconStorage(*config)
70+
return NewBeaconStorage(*config, db)
7271
}
7372

7473
type entry struct {

‎cmd/shisui/main.go

+8-11
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ import (
3939
"github.com/optimism-java/shisui2/portalwire"
4040
"github.com/optimism-java/shisui2/state"
4141
"github.com/optimism-java/shisui2/storage"
42-
"github.com/optimism-java/shisui2/storage/sqlite"
42+
"github.com/optimism-java/shisui2/storage/pebble"
4343
"github.com/optimism-java/shisui2/web3"
4444
"github.com/protolambda/zrnt/eth2/configs"
4545
"github.com/urfave/cli/v2"
@@ -379,16 +379,15 @@ func doPortMapping(natm nat.Interface, ln *enode.LocalNode, addr *net.UDPAddr) {
379379

380380
func initHistory(config Config, server *rpc.Server, conn discover.UDPConn, localNode *enode.LocalNode, discV5 *discover.UDPv5, utp *portalwire.PortalUtp) (*history.Network, error) {
381381
networkName := portalwire.History.Name()
382-
db, err := sqlite.NewDB(config.DataDir, networkName)
382+
db, err := pebble.NewDB(config.DataDir, 16, 400, networkName)
383383
if err != nil {
384384
return nil, err
385385
}
386-
contentStorage, err := sqlite.NewHistoryStorage(storage.PortalStorageConfig{
386+
contentStorage, err := pebble.NewStorage(storage.PortalStorageConfig{
387387
StorageCapacityMB: config.DataCapacity,
388-
DB: db,
389388
NodeId: localNode.ID(),
390389
NetworkName: networkName,
391-
})
390+
}, db)
392391
if err != nil {
393392
return nil, err
394393
}
@@ -435,11 +434,10 @@ func initBeacon(config Config, server *rpc.Server, conn discover.UDPConn, localN
435434

436435
contentStorage, err := beacon.NewBeaconStorage(storage.PortalStorageConfig{
437436
StorageCapacityMB: config.DataCapacity,
438-
DB: sqlDb,
439437
NodeId: localNode.ID(),
440438
Spec: configs.Mainnet,
441439
NetworkName: portalwire.Beacon.Name(),
442-
})
440+
}, sqlDb)
443441
if err != nil {
444442
return nil, err
445443
}
@@ -473,16 +471,15 @@ func initBeacon(config Config, server *rpc.Server, conn discover.UDPConn, localN
473471

474472
func initState(config Config, server *rpc.Server, conn discover.UDPConn, localNode *enode.LocalNode, discV5 *discover.UDPv5, utp *portalwire.PortalUtp) (*state.Network, error) {
475473
networkName := portalwire.State.Name()
476-
db, err := sqlite.NewDB(config.DataDir, networkName)
474+
db, err := pebble.NewDB(config.DataDir, 16, 400, networkName)
477475
if err != nil {
478476
return nil, err
479477
}
480-
contentStorage, err := sqlite.NewHistoryStorage(storage.PortalStorageConfig{
478+
contentStorage, err := pebble.NewStorage(storage.PortalStorageConfig{
481479
StorageCapacityMB: config.DataCapacity,
482-
DB: db,
483480
NodeId: localNode.ID(),
484481
NetworkName: networkName,
485-
})
482+
}, db)
486483
if err != nil {
487484
return nil, err
488485
}

‎state/storage.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ package state
33
import (
44
"bytes"
55
"crypto/sha256"
6-
"database/sql"
76
"errors"
87

8+
"github.com/cockroachdb/pebble"
99
"github.com/ethereum/go-ethereum/crypto"
1010
"github.com/ethereum/go-ethereum/log"
1111
"github.com/ethereum/go-ethereum/metrics"
@@ -24,24 +24,24 @@ var _ storage.ContentStorage = &Storage{}
2424

2525
type Storage struct {
2626
store storage.ContentStorage
27-
db *sql.DB
27+
db *pebble.DB
2828
log log.Logger
2929
}
3030

3131
var portalStorageMetrics *portalwire.PortalStorageMetrics
3232

33-
func NewStateStorage(store storage.ContentStorage, db *sql.DB) *Storage {
33+
func NewStateStorage(store storage.ContentStorage, db *pebble.DB) *Storage {
3434
stateStorage := &Storage{
3535
store: store,
3636
db: db,
3737
log: log.New("storage", "state"),
3838
}
3939

40-
var err error
41-
portalStorageMetrics, err = portalwire.NewPortalStorageMetrics("state", db)
42-
if err != nil {
43-
return nil
44-
}
40+
// var err error
41+
// portalStorageMetrics, err = portalwire.NewPortalStorageMetrics("state", db)
42+
// if err != nil {
43+
// return nil
44+
// }
4545

4646
return stateStorage
4747
}

‎storage/config.go

-3
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,12 @@
11
package storage
22

33
import (
4-
"database/sql"
5-
64
"github.com/ethereum/go-ethereum/p2p/enode"
75
"github.com/protolambda/zrnt/eth2/beacon/common"
86
)
97

108
type PortalStorageConfig struct {
119
StorageCapacityMB uint64
12-
DB *sql.DB
1310
NodeId enode.ID
1411
Spec *common.Spec
1512
NetworkName string

‎storage/pebble/storage.go

+4-11
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package pepple
1+
package pebble
22

33
import (
44
"bytes"
@@ -31,14 +31,7 @@ const (
3131

3232
var _ storage.ContentStorage = &ContentStorage{}
3333

34-
type PeppleStorageConfig struct {
35-
StorageCapacityMB uint64
36-
DB *pebble.DB
37-
NodeId enode.ID
38-
NetworkName string
39-
}
40-
41-
func NewPeppleDB(dataDir string, cache, handles int, namespace string) (*pebble.DB, error) {
34+
func NewDB(dataDir string, cache, handles int, namespace string) (*pebble.DB, error) {
4235
// Ensure we have some minimal caching and file guarantees
4336
if cache < minCache {
4437
cache = minCache
@@ -128,10 +121,10 @@ type ContentStorage struct {
128121
bytePool sync.Pool
129122
}
130123

131-
func NewPeppleStorage(config PeppleStorageConfig) (storage.ContentStorage, error) {
124+
func NewStorage(config storage.PortalStorageConfig, db *pebble.DB) (storage.ContentStorage, error) {
132125
cs := &ContentStorage{
133126
nodeId: config.NodeId,
134-
db: config.DB,
127+
db: db,
135128
storageCapacityInBytes: config.StorageCapacityMB * 1000_000,
136129
log: log.New("storage", config.NetworkName),
137130
writeOptions: &pebble.WriteOptions{Sync: false},

‎storage/pebble/storage_test.go

+5-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package pepple
1+
package pebble
22

33
import (
44
"testing"
@@ -18,26 +18,25 @@ func genBytes(length int) []byte {
1818
}
1919

2020
func TestNewPeppleDB(t *testing.T) {
21-
db, err := NewPeppleDB(t.TempDir(), 16, 16, "test")
21+
db, err := NewDB(t.TempDir(), 16, 16, "test")
2222
assert.NoError(t, err)
2323
defer db.Close()
2424

2525
assert.NotNil(t, db)
2626
}
2727

2828
func setupTestStorage(t *testing.T) storage.ContentStorage {
29-
db, err := NewPeppleDB(t.TempDir(), 16, 16, "test")
29+
db, err := NewDB(t.TempDir(), 16, 16, "test")
3030
assert.NoError(t, err)
3131
t.Cleanup(func() { db.Close() })
3232

33-
config := PeppleStorageConfig{
33+
config := storage.PortalStorageConfig{
3434
StorageCapacityMB: 1,
35-
DB: db,
3635
NodeId: uint256.NewInt(0).Bytes32(),
3736
NetworkName: "test",
3837
}
3938

40-
storage, err := NewPeppleStorage(config)
39+
storage, err := NewStorage(config, db)
4140
assert.NoError(t, err)
4241
return storage
4342
}

‎storage/sqlite/storage.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -104,10 +104,10 @@ func NewDB(dataDir string, network string) (*sql.DB, error) {
104104
return sqlDb, err
105105
}
106106

107-
func NewHistoryStorage(config storage.PortalStorageConfig) (storage.ContentStorage, error) {
107+
func NewStorage(config storage.PortalStorageConfig, db *sql.DB) (storage.ContentStorage, error) {
108108
hs := &ContentStorage{
109109
nodeId: config.NodeId,
110-
sqliteDB: config.DB,
110+
sqliteDB: db,
111111
storageCapacityInBytes: config.StorageCapacityMB * 1000000,
112112
log: log.New("storage", config.NetworkName),
113113
}
@@ -124,7 +124,7 @@ func NewHistoryStorage(config storage.PortalStorageConfig) (storage.ContentStora
124124

125125
// necessary to test NetworkName==history because state also initialize HistoryStorage
126126
if strings.ToLower(config.NetworkName) == "history" {
127-
portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, config.DB)
127+
portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, db)
128128
if err != nil {
129129
return nil, err
130130
}

‎storage/sqlite/storage_test.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,10 @@ func newContentStorage(storageCapacityInMB uint64, nodeId enode.ID) (*ContentSto
3131
if err != nil {
3232
return nil, err
3333
}
34-
hs, err := NewHistoryStorage(contentStorage.PortalStorageConfig{
35-
DB: db,
34+
hs, err := NewStorage(contentStorage.PortalStorageConfig{
3635
StorageCapacityMB: storageCapacityInMB,
3736
NodeId: nodeId,
38-
})
37+
}, db)
3938
if err != nil {
4039
return nil, err
4140
}

‎storage/tests/storage_bench_test.go

+5-7
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,12 @@ func BenchmarkStorageComparison(b *testing.B) {
4848

4949
b.Run("Pebble_"+tc.name, func(b *testing.B) {
5050
dir, _ := os.MkdirTemp("", "pebble-bench-*")
51-
db, _ := ethpepple.NewPeppleDB(dir, 16, 16, "bench")
52-
storage, _ := ethpepple.NewPeppleStorage(ethpepple.PeppleStorageConfig{
51+
db, _ := ethpepple.NewDB(dir, 16, 16, "bench")
52+
storage, _ := ethpepple.NewStorage(storage.PortalStorageConfig{
5353
StorageCapacityMB: 1000,
54-
DB: db,
5554
NodeId: enode.ID{},
5655
NetworkName: "bench",
57-
})
56+
}, db)
5857
defer func() {
5958
db.Close()
6059
os.RemoveAll(dir)
@@ -77,12 +76,11 @@ func BenchmarkStorageComparison(b *testing.B) {
7776
b.Run("SQLite_"+tc.name, func(b *testing.B) {
7877
dir, _ := os.MkdirTemp("", "sqlite-bench-*")
7978
db, _ := sqlite.NewDB(dir, "bench")
80-
storage, _ := sqlite.NewHistoryStorage(storage.PortalStorageConfig{
79+
storage, _ := sqlite.NewStorage(storage.PortalStorageConfig{
8180
StorageCapacityMB: 1000,
82-
DB: db,
8381
NodeId: enode.ID{},
8482
NetworkName: "bench",
85-
})
83+
}, db)
8684
defer func() {
8785
storage.Close()
8886
os.RemoveAll(dir)

0 commit comments

Comments
 (0)