implement DB options for customizing maxopenfiles backport from cosmo…

…s-db(#51)

backend_test.go

@@ -2,7 +2,6 @@ package db
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -14,7 +13,7 @@ import (
 // Register a test backend for PrefixDB as well, with some unrelated junk data
 func init() {
 	// nolint: errcheck
-	registerDBCreator("prefixdb", func(name, dir string) (DB, error) {
+	registerDBCreator("prefixdb", func(name, dir string, opts Options) (DB, error) {
 		mdb := NewMemDB()
 		mdb.Set([]byte("a"), []byte{1})
 		mdb.Set([]byte("b"), []byte{2})
@@ -35,11 +34,11 @@ func cleanupDBDir(dir, name string) {
 
 func testBackendGetSetDelete(t *testing.T, backend BackendType) {
 	// Default
-	dirname, err := ioutil.TempDir("", fmt.Sprintf("test_backend_%s_", backend))
-	require.Nil(t, err)
-	db, err := NewDB("testdb", backend, dirname)
+	dbName := "testdb"
+	dir := os.TempDir()
+	db, err := NewDB(dbName, backend, dir)
 	require.NoError(t, err)
-	defer cleanupDBDir(dirname, "testdb")
+	defer os.RemoveAll(fmt.Sprintf("%s/%s.db", dir, dbName))
 
 	// A nonexistent key should return nil.
 	value, err := db.Get([]byte("a"))
@@ -166,7 +165,7 @@ func testDBIterator(t *testing.T, backend BackendType) {
 	dir := os.TempDir()
 	db, err := NewDB(name, backend, dir)
 	require.NoError(t, err)
-	defer cleanupDBDir(dir, name)
+	defer os.RemoveAll(fmt.Sprintf("%s/%s.db", dir, name))
 
 	for i := 0; i < 10; i++ {
 		if i != 6 { // but skip 6.
@@ -302,11 +301,11 @@ func testDBIterator(t *testing.T, backend BackendType) {
 		[]int64(nil), "reverse iterator from 2 (ex) to 4")
 
 	// Ensure that the iterators don't panic with an empty database.
-	dir2, err := ioutil.TempDir("", "tm-db-test")
 	require.NoError(t, err)
-	db2, err := NewDB(name, backend, dir2)
+	name2 := fmt.Sprintf("test_%x", randStr(11))
+	db2, err := NewDB(name2, backend, dir)
 	require.NoError(t, err)
-	defer cleanupDBDir(dir2, name)
+	defer os.RemoveAll(fmt.Sprintf("%s/%s.db", dir, name2))
 
 	itr, err = db2.Iterator(nil, nil)
 	require.NoError(t, err)

badger_db.go

@@ -14,7 +14,7 @@ import (
 
 func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) }
 
-func badgerDBCreator(dbName, dir string) (DB, error) {
+func badgerDBCreator(dbName, dir string, opts Options) (DB, error) {
 	return NewBadgerDB(dbName, dir)
 }
 

boltdb.go

@@ -32,7 +32,7 @@ type BoltDB struct {
 var _ DB = (*BoltDB)(nil)
 
 // NewBoltDB returns a BoltDB with default options.
-func NewBoltDB(name, dir string) (DB, error) {
+func NewBoltDB(name, dir string, opts Options) (DB, error) {
 	return NewBoltDBWithOpts(name, dir, bbolt.DefaultOptions)
 }
 

boltdb_test.go

@@ -17,7 +17,7 @@ func TestBoltDBNewBoltDB(t *testing.T) {
 	dir := os.TempDir()
 	defer cleanupDBDir(dir, name)
 
-	db, err := NewBoltDB(name, dir)
+	db, err := NewBoltDB(name, dir, nil)
 	require.NoError(t, err)
 	db.Close()
 }
@@ -26,15 +26,15 @@ func TestWithBoltDB(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "boltdb")
 
-	db, err := NewBoltDB(path, "")
+	db, err := NewBoltDB(path, "", nil)
 	require.NoError(t, err)
 
 	t.Run("BoltDB", func(t *testing.T) { Run(t, db) })
 }
 
 func BenchmarkBoltDBRandomReadsWrites(b *testing.B) {
 	name := fmt.Sprintf("test_%x", randStr(12))
-	db, err := NewBoltDB(name, "")
+	db, err := NewBoltDB(name, "", nil)
 	if err != nil {
 		b.Fatal(err)
 	}

cleveldb.go

@@ -8,11 +8,12 @@ import (
 	"path/filepath"
 
 	"github.com/jmhodges/levigo"
+	"github.com/spf13/cast"
 )
 
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
-		return NewCLevelDB(name, dir)
+	dbCreator := func(name string, dir string, opts Options) (DB, error) {
+		return NewCLevelDB(name, dir, opts)
 	}
 	registerDBCreator(CLevelDBBackend, dbCreator, false)
 }
@@ -27,14 +28,26 @@ type CLevelDB struct {
 
 var _ DB = (*CLevelDB)(nil)
 
-// NewCLevelDB creates a new CLevelDB.
-func NewCLevelDB(name string, dir string) (*CLevelDB, error) {
-	dbPath := filepath.Join(dir, name+".db")
-
+func defaultCleveldbOptions() *levigo.Options {
 	opts := levigo.NewOptions()
 	opts.SetCache(levigo.NewLRUCache(1 << 30))
 	opts.SetCreateIfMissing(true)
-	db, err := levigo.Open(dbPath, opts)
+	return opts
+}
+
+// NewCLevelDB creates a new CLevelDB.
+func NewCLevelDB(name string, dir string, opts Options) (*CLevelDB, error) {
+	do := defaultCleveldbOptions()
+
+	if opts != nil {
+		files := cast.ToInt(opts.Get("maxopenfiles"))
+		if files > 0 {
+			do.SetMaxOpenFiles(files)
+		}
+	}
+
+	dbPath := filepath.Join(dir, name+".db")
+	db, err := levigo.Open(dbPath, do)
 	if err != nil {
 		return nil, err
 	}

cleveldb_test.go

@@ -19,13 +19,13 @@ func TestWithClevelDB(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "cleveldb")
 
-	db, err := NewCLevelDB(path, "")
+	db, err := NewCLevelDB(path, "", nil)
 	require.NoError(t, err)
 
 	t.Run("ClevelDB", func(t *testing.T) { Run(t, db) })
 }
 
-//nolint: errcheck
+// nolint: errcheck
 func BenchmarkRandomReadsWrites2(b *testing.B) {
 	b.StopTimer()
 
@@ -34,7 +34,7 @@ func BenchmarkRandomReadsWrites2(b *testing.B) {
 	for i := 0; i < int(numItems); i++ {
 		internal[int64(i)] = int64(0)
 	}
-	db, err := NewCLevelDB(fmt.Sprintf("test_%x", randStr(12)), "")
+	db, err := NewCLevelDB(fmt.Sprintf("test_%x", randStr(12)), "", nil)
 	if err != nil {
 		b.Fatal(err.Error())
 		return

db.go

@@ -37,7 +37,13 @@ const (
 	BadgerDBBackend BackendType = "badgerdb"
 )
 
-type dbCreator func(name string, dir string) (DB, error)
+type (
+	dbCreator func(name string, dir string, opts Options) (DB, error)
+
+	Options interface {
+		Get(string) interface{}
+	}
+)
 
 var backends = map[BackendType]dbCreator{}
 
@@ -51,6 +57,10 @@ func registerDBCreator(backend BackendType, creator dbCreator, force bool) {
 
 // NewDB creates a new database of type backend with the given name.
 func NewDB(name string, backend BackendType, dir string) (DB, error) {
+	return NewDBwithOptions(name, backend, dir, nil)
+}
+
+func NewDBwithOptions(name string, backend BackendType, dir string, opts Options) (DB, error) {
 	dbCreator, ok := backends[backend]
 	if !ok {
 		keys := make([]string, 0, len(backends))
@@ -61,7 +71,7 @@ func NewDB(name string, backend BackendType, dir string) (DB, error) {
 			backend, strings.Join(keys, ","))
 	}
 
-	db, err := dbCreator(name, dir)
+	db, err := dbCreator(name, dir, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize database: %w", err)
 	}

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/btree v1.1.2
 	github.com/jmhodges/levigo v1.0.0
+	github.com/spf13/cast v1.3.0
 	github.com/stretchr/testify v1.8.0
 	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
 	go.etcd.io/bbolt v1.3.6

go.sum

@@ -128,6 +128,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=

goleveldb.go

@@ -4,15 +4,16 @@ import (
 	"fmt"
 	"path/filepath"
 
+	"github.com/spf13/cast"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
-		return NewGoLevelDB(name, dir)
+	dbCreator := func(name string, dir string, opts Options) (DB, error) {
+		return OpenGoLevelDBWithOpts(name, dir, opts)
 	}
 	registerDBCreator(GoLevelDBBackend, dbCreator, false)
 }
@@ -23,10 +24,29 @@ type GoLevelDB struct {
 
 var _ DB = (*GoLevelDB)(nil)
 
+// NewGoLevelDB open new go level database with default options.
+// Note: tendermint(v0.34.21) backend db is using this function,
+// it can be removed when tendermint discard the tm-db dependencies.
 func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) {
-	return NewGoLevelDBWithOpts(name, dir, nil)
+	return OpenGoLevelDBWithOpts(name, dir, nil)
 }
 
+// OpenGoLevelDBWithOpts open new go level database with default options
+// and custom options for the db clients.
+func OpenGoLevelDBWithOpts(name string, dir string, opts Options) (*GoLevelDB, error) {
+	defaultOpts := &opt.Options{}
+
+	if opts != nil {
+		files := cast.ToInt(opts.Get("maxopenfiles"))
+		if files > 0 {
+			defaultOpts.OpenFilesCacheCapacity = files
+		}
+	}
+
+	return NewGoLevelDBWithOpts(name, dir, defaultOpts)
+}
+
+// NewGoLevelDBWithOpts open new go level database with leveldb options.
 func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) {
 	dbPath := filepath.Join(dir, name+".db")
 	db, err := leveldb.OpenFile(dbPath, o)

memdb.go

@@ -14,7 +14,7 @@ const (
 )
 
 func init() {
-	registerDBCreator(MemDBBackend, func(name, dir string) (DB, error) {
+	registerDBCreator(MemDBBackend, func(name, dir string, opts Options) (DB, error) {
 		return NewMemDB(), nil
 	}, false)
 }

rocksdb.go

@@ -9,11 +9,12 @@ import (
 	"runtime"
 
 	"github.com/cosmos/gorocksdb"
+	"github.com/spf13/cast"
 )
 
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
-		return NewRocksDB(name, dir)
+	dbCreator := func(name string, dir string, opts Options) (DB, error) {
+		return NewRocksDB(name, dir, opts)
 	}
 	registerDBCreator(RocksDBBackend, dbCreator, false)
 }
@@ -28,23 +29,36 @@ type RocksDB struct {
 
 var _ DB = (*RocksDB)(nil)
 
-func NewRocksDB(name string, dir string) (*RocksDB, error) {
-	// default rocksdb option, good enough for most cases, including heavy workloads.
-	// 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads).
-	// compression: snappy as default, need to -lsnappy to enable.
+// defaultRocksdbOptions, good enough for most cases, including heavy workloads.
+// 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads).
+// compression: snappy as default, need to -lsnappy to enable.
+func defaultRocksdbOptions() *gorocksdb.Options {
 	bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
 	bbto.SetBlockCache(gorocksdb.NewLRUCache(1 << 30))
 	bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(10))
 
-	opts := gorocksdb.NewDefaultOptions()
-	opts.SetBlockBasedTableFactory(bbto)
+	rocksdbOpts := gorocksdb.NewDefaultOptions()
+	rocksdbOpts.SetBlockBasedTableFactory(bbto)
 	// SetMaxOpenFiles to 4096 seems to provide a reliable performance boost
-	opts.SetMaxOpenFiles(4096)
-	opts.SetCreateIfMissing(true)
-	opts.IncreaseParallelism(runtime.NumCPU())
+	rocksdbOpts.SetMaxOpenFiles(4096)
+	rocksdbOpts.SetCreateIfMissing(true)
+	rocksdbOpts.IncreaseParallelism(runtime.NumCPU())
 	// 1.5GB maximum memory use for writebuffer.
-	opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
-	return NewRocksDBWithOptions(name, dir, opts)
+	rocksdbOpts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
+	return rocksdbOpts
+}
+
+func NewRocksDB(name string, dir string, opts Options) (*RocksDB, error) {
+	defaultOpts := defaultRocksdbOptions()
+
+	if opts != nil {
+		files := cast.ToInt(opts.Get("maxopenfiles"))
+		if files > 0 {
+			defaultOpts.SetMaxOpenFiles(files)
+		}
+	}
+
+	return NewRocksDBWithOptions(name, dir, defaultOpts)
 }
 
 func NewRocksDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksDB, error) {

rocksdb_test.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/spf13/cast"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -28,7 +29,7 @@ func TestWithRocksDB(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "rocksdb")
 
-	db, err := NewRocksDB(path, "")
+	db, err := NewRocksDB(path, "", nil)
 	require.NoError(t, err)
 
 	t.Run("RocksDB", func(t *testing.T) { Run(t, db) })
@@ -44,4 +45,20 @@ func TestRocksDBStats(t *testing.T) {
 	assert.NotEmpty(t, db.Stats())
 }
 
-// TODO: Add tests for rocksdb
+func TestRocksDBWithOptions(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "rocksdb")
+
+	opts := make(OptionsMap, 0)
+	opts["maxopenfiles"] = 1000
+
+	defaultOpts := defaultRocksdbOptions()
+	files := cast.ToInt(opts.Get("maxopenfiles"))
+	defaultOpts.SetMaxOpenFiles(files)
+	require.Equal(t, opts["maxopenfiles"], defaultOpts.GetMaxOpenFiles())
+
+	db, err := NewRocksDB(path, "", opts)
+	require.NoError(t, err)
+
+	t.Run("RocksDB", func(t *testing.T) { Run(t, db) })
+}