Skip to content
This repository was archived by the owner on Mar 8, 2025. It is now read-only.

Commit f263439

Browse files
ethenotethanethenotethan
authored
fix(Layr-Labs#191): Routing namespace --> Storage (Layr-Labs#195)
1 parent 390e491 commit f263439

File tree

24 files changed

+474
-354
lines changed

24 files changed

+474
-354
lines changed

‎README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,9 @@ In order to disperse to the EigenDA network in production, or at high throughput
6363
| `--s3.path` | | `$EIGENDA_PROXY_S3_PATH` | Bucket path for S3 storage. |
6464
| `--s3.endpoint` | | `$EIGENDA_PROXY_S3_ENDPOINT` | Endpoint for S3 storage. |
6565
| `--s3.enable-tls` | | `$EIGENDA_PROXY_S3_ENABLE_TLS` | Enable TLS connection to S3 endpoint. |
66-
| `--routing.fallback-targets` | `[]` | `$EIGENDA_PROXY_FALLBACK_TARGETS` | Fall back backend targets. Supports S3. | Backup storage locations to read from in the event of eigenda retrieval failure. |
67-
| `--routing.cache-targets` | `[]` | `$EIGENDA_PROXY_CACHE_TARGETS` | Caching targets. Supports S3. | Caches data to backend targets after dispersing to DA, retrieved from before trying read from EigenDA. |
68-
| `--routing.concurrent-write-threads` | `0` | `$EIGENDA_PROXY_CONCURRENT_WRITE_THREADS` | Number of threads spun-up for async secondary storage insertions. (<=0) denotes single threaded insertions where (>0) indicates decoupled writes. |
66+
| `--storage.fallback-targets` | `[]` | `$EIGENDA_PROXY_STORAGE_FALLBACK_TARGETS` | Fall back backend targets. Supports S3. | Backup storage locations to read from in the event of eigenda retrieval failure. |
67+
| `--storage.cache-targets` | `[]` | `$EIGENDA_PROXY_STORAGE_CACHE_TARGETS` | Caching targets. Supports S3. | Caches data to backend targets after dispersing to DA, retrieved from before trying read from EigenDA. |
68+
| `--storage.concurrent-write-threads` | `0` | `$EIGENDA_PROXY_STORAGE_CONCURRENT_WRITE_THREADS` | Number of threads spun-up for async secondary storage insertions. (<=0) denotes single threaded insertions where (>0) indicates decoupled writes. |
6969
| `--s3.timeout` | `5s` | `$EIGENDA_PROXY_S3_TIMEOUT` | timeout for S3 storage operations (e.g. get, put) |
7070
| `--redis.db` | `0` | `$EIGENDA_PROXY_REDIS_DB` | redis database to use after connecting to server |
7171
| `--redis.endpoint` | `""` | `$EIGENDA_PROXY_REDIS_ENDPOINT` | redis endpoint url |

‎utils/utils.go renamed to ‎common/common.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package utils
1+
package common
22

33
import (
44
"fmt"

‎utils/utils_test.go renamed to ‎common/common_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
package utils_test
1+
package common_test
22

33
import (
44
"fmt"
55
"testing"
66

7-
"github.com/Layr-Labs/eigenda-proxy/utils"
7+
"github.com/Layr-Labs/eigenda-proxy/common"
88
)
99

1010
func TestParseByteAmount(t *testing.T) {
@@ -46,7 +46,7 @@ func TestParseByteAmount(t *testing.T) {
4646
t.Run(fmt.Sprintf("Input: %s", tc.input), func(t *testing.T) {
4747
t.Parallel()
4848

49-
got, err := utils.ParseBytesAmount(tc.input)
49+
got, err := common.ParseBytesAmount(tc.input)
5050
if (err != nil) != tc.wantErr {
5151
t.Errorf("wantErr: %v, got error: %v", tc.wantErr, err)
5252
}

‎common/store.go

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package common
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
)
8+
9+
// BackendType ... Storage backend type
10+
type BackendType uint8
11+
12+
const (
13+
EigenDABackendType BackendType = iota
14+
MemoryBackendType
15+
S3BackendType
16+
RedisBackendType
17+
18+
UnknownBackendType
19+
)
20+
21+
var (
22+
ErrProxyOversizedBlob = fmt.Errorf("encoded blob is larger than max blob size")
23+
ErrEigenDAOversizedBlob = fmt.Errorf("blob size cannot exceed")
24+
)
25+
26+
func (b BackendType) String() string {
27+
switch b {
28+
case EigenDABackendType:
29+
return "EigenDA"
30+
case MemoryBackendType:
31+
return "Memory"
32+
case S3BackendType:
33+
return "S3"
34+
case RedisBackendType:
35+
return "Redis"
36+
case UnknownBackendType:
37+
fallthrough
38+
default:
39+
return "Unknown"
40+
}
41+
}
42+
43+
func StringToBackendType(s string) BackendType {
44+
lower := strings.ToLower(s)
45+
46+
switch lower {
47+
case "eigenda":
48+
return EigenDABackendType
49+
case "memory":
50+
return MemoryBackendType
51+
case "s3":
52+
return S3BackendType
53+
case "redis":
54+
return RedisBackendType
55+
case "unknown":
56+
fallthrough
57+
default:
58+
return UnknownBackendType
59+
}
60+
}
61+
62+
type Store interface {
63+
// Backend returns the backend type provider of the store.
64+
BackendType() BackendType
65+
// Verify verifies the given key-value pair.
66+
Verify(ctx context.Context, key []byte, value []byte) error
67+
}
68+
69+
type GeneratedKeyStore interface {
70+
Store
71+
// Get retrieves the given key if it's present in the key-value data store.
72+
Get(ctx context.Context, key []byte) ([]byte, error)
73+
// Put inserts the given value into the key-value data store.
74+
Put(ctx context.Context, value []byte) (key []byte, err error)
75+
}
76+
77+
type PrecomputedKeyStore interface {
78+
Store
79+
// Get retrieves the given key if it's present in the key-value data store.
80+
Get(ctx context.Context, key []byte) ([]byte, error)
81+
// Put inserts the given value into the key-value data store.
82+
Put(ctx context.Context, key []byte, value []byte) error
83+
}

‎e2e/main_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77

88
"github.com/Layr-Labs/eigenda-proxy/client"
99
"github.com/Layr-Labs/eigenda-proxy/commitments"
10+
"github.com/Layr-Labs/eigenda-proxy/common"
1011
"github.com/Layr-Labs/eigenda-proxy/e2e"
1112
"github.com/Layr-Labs/eigenda-proxy/store"
1213
altda "github.com/ethereum-optimism/optimism/op-alt-da"
@@ -54,7 +55,7 @@ func requireDispersalRetrievalEigenDA(t *testing.T, cm *metrics.CountMap, mode c
5455
}
5556

5657
// requireWriteReadSecondary ... ensure that secondary backend was successfully written/read to/from
57-
func requireWriteReadSecondary(t *testing.T, cm *metrics.CountMap, bt store.BackendType) {
58+
func requireWriteReadSecondary(t *testing.T, cm *metrics.CountMap, bt common.BackendType) {
5859
writeCount, err := cm.Get(http.MethodPut, store.Success, bt.String())
5960
require.NoError(t, err)
6061
require.True(t, writeCount > 0)

‎e2e/safety_checks_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ func TestKeccak256CommitmentRequestErrorsWhenS3NotSet(t *testing.T) {
145145
testCfg.UseKeccak256ModeS3 = true
146146

147147
tsConfig := e2e.TestSuiteConfig(testCfg)
148-
tsConfig.EigenDAConfig.S3Config.Endpoint = ""
148+
tsConfig.EigenDAConfig.StorageConfig.S3Config.Endpoint = ""
149149
ts, kill := e2e.CreateTestSuite(tsConfig)
150150
defer kill()
151151

‎e2e/server_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66

77
"github.com/Layr-Labs/eigenda-proxy/client"
88
"github.com/Layr-Labs/eigenda-proxy/commitments"
9-
"github.com/Layr-Labs/eigenda-proxy/store"
9+
"github.com/Layr-Labs/eigenda-proxy/common"
1010
"github.com/stretchr/testify/require"
1111

1212
"github.com/Layr-Labs/eigenda-proxy/e2e"
@@ -100,7 +100,7 @@ func TestProxyCaching(t *testing.T) {
100100
defer kill()
101101

102102
requireSimpleClientSetGet(t, ts, e2e.RandBytes(1_000_000))
103-
requireWriteReadSecondary(t, ts.Metrics.SecondaryRequestsTotal, store.S3BackendType)
103+
requireWriteReadSecondary(t, ts.Metrics.SecondaryRequestsTotal, common.S3BackendType)
104104
requireDispersalRetrievalEigenDA(t, ts.Metrics.HTTPServerRequestsTotal, commitments.SimpleCommitmentMode)
105105
}
106106

@@ -119,7 +119,7 @@ func TestProxyCachingWithRedis(t *testing.T) {
119119
defer kill()
120120

121121
requireSimpleClientSetGet(t, ts, e2e.RandBytes(1_000_000))
122-
requireWriteReadSecondary(t, ts.Metrics.SecondaryRequestsTotal, store.RedisBackendType)
122+
requireWriteReadSecondary(t, ts.Metrics.SecondaryRequestsTotal, common.RedisBackendType)
123123
requireDispersalRetrievalEigenDA(t, ts.Metrics.HTTPServerRequestsTotal, commitments.SimpleCommitmentMode)
124124
}
125125

@@ -163,6 +163,6 @@ func TestProxyReadFallback(t *testing.T) {
163163
require.Equal(t, expectedBlob, actualBlob)
164164

165165
requireSimpleClientSetGet(t, ts, e2e.RandBytes(1_000_000))
166-
requireWriteReadSecondary(t, ts.Metrics.SecondaryRequestsTotal, store.S3BackendType)
166+
requireWriteReadSecondary(t, ts.Metrics.SecondaryRequestsTotal, common.S3BackendType)
167167
requireDispersalRetrievalEigenDA(t, ts.Metrics.HTTPServerRequestsTotal, commitments.SimpleCommitmentMode)
168168
}

‎e2e/setup.go

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@ import (
88
"strings"
99
"time"
1010

11+
"github.com/Layr-Labs/eigenda-proxy/common"
1112
"github.com/Layr-Labs/eigenda-proxy/metrics"
1213
"github.com/Layr-Labs/eigenda-proxy/server"
14+
"github.com/Layr-Labs/eigenda-proxy/store"
1315
"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
1416
"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
1517
"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
16-
"github.com/Layr-Labs/eigenda-proxy/utils"
1718
"github.com/Layr-Labs/eigenda-proxy/verify"
1819
"github.com/Layr-Labs/eigenda/api/clients"
1920
"github.com/Layr-Labs/eigenda/encoding/kzg"
@@ -127,7 +128,7 @@ func TestConfig(useMemory bool) *Cfg {
127128
}
128129

129130
func createRedisConfig(eigendaCfg server.Config) server.CLIConfig {
130-
eigendaCfg.RedisConfig = redis.Config{
131+
eigendaCfg.StorageConfig.RedisConfig = redis.Config{
131132
Endpoint: redisEndpoint,
132133
Password: "",
133134
DB: 0,
@@ -144,7 +145,7 @@ func createS3Config(eigendaCfg server.Config) server.CLIConfig {
144145
bucketName := "eigenda-proxy-test-" + RandStr(10)
145146
createS3Bucket(bucketName)
146147

147-
eigendaCfg.S3Config = s3.Config{
148+
eigendaCfg.StorageConfig.S3Config = s3.Config{
148149
Bucket: bucketName,
149150
Path: "",
150151
Endpoint: minioEndpoint,
@@ -178,7 +179,7 @@ func TestSuiteConfig(testCfg *Cfg) server.CLIConfig {
178179
pollInterval = time.Minute * 1
179180
}
180181

181-
maxBlobLengthBytes, err := utils.ParseBytesAmount("16mib")
182+
maxBlobLengthBytes, err := common.ParseBytesAmount("16mib")
182183
if err != nil {
183184
panic(err)
184185
}
@@ -213,7 +214,10 @@ func TestSuiteConfig(testCfg *Cfg) server.CLIConfig {
213214
BlobExpiration: testCfg.Expiration,
214215
MaxBlobSizeBytes: maxBlobLengthBytes,
215216
},
216-
AsyncPutWorkers: testCfg.WriteThreadCount,
217+
218+
StorageConfig: store.Config{
219+
AsyncPutWorkers: testCfg.WriteThreadCount,
220+
},
217221
}
218222

219223
if testCfg.UseMemory {
@@ -226,15 +230,15 @@ func TestSuiteConfig(testCfg *Cfg) server.CLIConfig {
226230
cfg = createS3Config(eigendaCfg)
227231

228232
case testCfg.UseS3Caching:
229-
eigendaCfg.CacheTargets = []string{"S3"}
233+
eigendaCfg.StorageConfig.CacheTargets = []string{"S3"}
230234
cfg = createS3Config(eigendaCfg)
231235

232236
case testCfg.UseS3Fallback:
233-
eigendaCfg.FallbackTargets = []string{"S3"}
237+
eigendaCfg.StorageConfig.FallbackTargets = []string{"S3"}
234238
cfg = createS3Config(eigendaCfg)
235239

236240
case testCfg.UseRedisCaching:
237-
eigendaCfg.CacheTargets = []string{"redis"}
241+
eigendaCfg.StorageConfig.CacheTargets = []string{"redis"}
238242
cfg = createRedisConfig(eigendaCfg)
239243

240244
default:

‎flags/flags.go

Lines changed: 5 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package flags
22

33
import (
44
"github.com/Layr-Labs/eigenda-proxy/flags/eigendaflags"
5+
"github.com/Layr-Labs/eigenda-proxy/store"
56
"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
67
"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
78
"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
@@ -17,6 +18,8 @@ const (
1718
EigenDAClientCategory = "EigenDA Client"
1819
EigenDADeprecatedCategory = "DEPRECATED EIGENDA CLIENT FLAGS -- THESE WILL BE REMOVED IN V2.0.0"
1920
MemstoreFlagsCategory = "Memstore (for testing purposes - replaces EigenDA backend)"
21+
StorageFlagsCategory = "Storage"
22+
StorageDeprecatedCategory = "DEPRECATED STORAGE FLAGS -- THESE WILL BE REMOVED IN V2.0.0"
2023
RedisCategory = "Redis Cache/Fallback"
2124
S3Category = "S3 Cache/Fallback"
2225
VerifierCategory = "KZG and Cert Verifier"
@@ -26,12 +29,6 @@ const (
2629
const (
2730
ListenAddrFlagName = "addr"
2831
PortFlagName = "port"
29-
30-
// routing flags
31-
// TODO: change "routing" --> "secondary"
32-
FallbackTargetsFlagName = "routing.fallback-targets"
33-
CacheTargetsFlagName = "routing.cache-targets"
34-
ConcurrentWriteThreads = "routing.concurrent-write-routines"
3532
)
3633

3734
const EnvVarPrefix = "EIGENDA_PROXY"
@@ -55,24 +52,6 @@ func CLIFlags() []cli.Flag {
5552
Value: 3100,
5653
EnvVars: prefixEnvVars("PORT"),
5754
},
58-
&cli.StringSliceFlag{
59-
Name: FallbackTargetsFlagName,
60-
Usage: "List of read fallback targets to rollover to if cert can't be read from EigenDA.",
61-
Value: cli.NewStringSlice(),
62-
EnvVars: prefixEnvVars("FALLBACK_TARGETS"),
63-
},
64-
&cli.StringSliceFlag{
65-
Name: CacheTargetsFlagName,
66-
Usage: "List of caching targets to use fast reads from EigenDA.",
67-
Value: cli.NewStringSlice(),
68-
EnvVars: prefixEnvVars("CACHE_TARGETS"),
69-
},
70-
&cli.IntFlag{
71-
Name: ConcurrentWriteThreads,
72-
Usage: "Number of threads spun-up for async secondary storage insertions. (<=0) denotes single threaded insertions where (>0) indicates decoupled writes.",
73-
Value: 0,
74-
EnvVars: prefixEnvVars("CONCURRENT_WRITE_THREADS"),
75-
},
7655
}
7756

7857
return flags
@@ -87,6 +66,8 @@ func init() {
8766
Flags = append(Flags, opmetrics.CLIFlags(EnvVarPrefix)...)
8867
Flags = append(Flags, eigendaflags.CLIFlags(EnvVarPrefix, EigenDAClientCategory)...)
8968
Flags = append(Flags, eigendaflags.DeprecatedCLIFlags(EnvVarPrefix, EigenDADeprecatedCategory)...)
69+
Flags = append(Flags, store.CLIFlags(EnvVarPrefix, StorageFlagsCategory)...)
70+
Flags = append(Flags, store.DeprecatedCLIFlags(EnvVarPrefix, StorageDeprecatedCategory)...)
9071
Flags = append(Flags, redis.CLIFlags(EnvVarPrefix, RedisCategory)...)
9172
Flags = append(Flags, s3.CLIFlags(EnvVarPrefix, S3Category)...)
9273
Flags = append(Flags, memstore.CLIFlags(EnvVarPrefix, MemstoreFlagsCategory)...)

0 commit comments

Comments
 (0)