Skip to content

Commit b99c8d2

Browse files
jbowensjbowens
committed
db: add TestLargeKeys
Add a new datadriven unit test that exercises large keys with a long common shared prefix. This test demonstrates the worst-case unbounded block sizes. Informs #4518.
1 parent aa0446a commit b99c8d2

File tree

4 files changed

+447
-23
lines changed

4 files changed

+447
-23
lines changed

‎data_test.go

+23
Original file line numberDiff line numberDiff line change
@@ -1321,6 +1321,29 @@ func runSSTablePropertiesCmd(t *testing.T, td *datadriven.TestData, d *DB) strin
13211321
return buf.String()
13221322
}
13231323

1324+
func runLayoutCmd(t *testing.T, td *datadriven.TestData, d *DB) string {
1325+
var filename string
1326+
td.ScanArgs(t, "filename", &filename)
1327+
f, err := d.opts.FS.Open(filename)
1328+
if err != nil {
1329+
return err.Error()
1330+
}
1331+
readable, err := sstable.NewSimpleReadable(f)
1332+
if err != nil {
1333+
return err.Error()
1334+
}
1335+
r, err := sstable.NewReader(context.Background(), readable, d.opts.MakeReaderOptions())
1336+
if err != nil {
1337+
return err.Error()
1338+
}
1339+
defer r.Close()
1340+
l, err := r.Layout()
1341+
if err != nil {
1342+
return err.Error()
1343+
}
1344+
return l.Describe(td.HasArg("verbose"), r, nil)
1345+
}
1346+
13241347
func runPopulateCmd(t *testing.T, td *datadriven.TestData, b *Batch) {
13251348
var maxKeyLength, valLength int
13261349
var timestamps []int

‎iterator_histories_test.go

+1-23
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ package pebble
55

66
import (
77
"bytes"
8-
"context"
98
"fmt"
109
"path/filepath"
1110
"strconv"
@@ -240,28 +239,7 @@ func TestIterHistories(t *testing.T) {
240239
}
241240
return ""
242241
case "layout":
243-
var verbose bool
244-
var filename string
245-
td.ScanArgs(t, "filename", &filename)
246-
td.MaybeScanArgs(t, "verbose", &verbose)
247-
f, err := opts.FS.Open(filename)
248-
if err != nil {
249-
return err.Error()
250-
}
251-
readable, err := sstable.NewSimpleReadable(f)
252-
if err != nil {
253-
return err.Error()
254-
}
255-
r, err := sstable.NewReader(context.Background(), readable, opts.MakeReaderOptions())
256-
if err != nil {
257-
return err.Error()
258-
}
259-
defer r.Close()
260-
l, err := r.Layout()
261-
if err != nil {
262-
return err.Error()
263-
}
264-
return l.Describe(verbose, r, nil)
242+
return runLayoutCmd(t, td, d)
265243
case "lsm":
266244
return runLSMCmd(td, d)
267245
case "metrics":

‎testdata/large_keys

+285
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,285 @@
1+
# FormatFlushableIngestExcises is format major version 18, which precedes
2+
# support for columnar blocks.
3+
#
4+
# Set a target file size of 4MB.
5+
6+
define target-file-sizes=(4000000) format-major-version=18
7+
----
8+
9+
# Commit many keys with 1MB-shared prefixes.
10+
11+
batch-commit
12+
set a(p,1000000)arition
13+
set a(p,1000000)alling
14+
set a(p,1000000)eal
15+
set a(p,1000000)ellate
16+
set a(p,1000000)endectomy
17+
set a(p,1000000)etizers
18+
set a(p,1000000)etizing
19+
set a(p,1000000)laude
20+
set a(p,1000000)lauding
21+
set a(p,1000000)le
22+
set a(p,1000000)les
23+
set a(p,1000000)letini
24+
set a(p,1000000)letinis
25+
set a(p,1000000)lebottomjeans
26+
set a(p,1000000)lication
27+
set a(p,1000000)ly
28+
set a(p,1000000)lying
29+
set a(p,1000000)ollo
30+
set a(p,1000000)raisal
31+
set a(p,1000000)raisals
32+
set a(p,1000000)raiser
33+
set a(p,1000000)raisers
34+
set a(p,1000000)raising
35+
set a(p,1000000)rentice
36+
set a(p,1000000)rentices
37+
set a(p,1000000)renticing
38+
set a(p,1000000)roval
39+
set a(p,1000000)rovals
40+
set a(p,1000000)rove
41+
----
42+
43+
flush verbose
44+
----
45+
L0.0:
46+
000005:[a(p,1000000)a(l,2)ing#11,SET-a(p,1000000)rove#38,SET] seqnums:[10-38] points:[a(p,1000000)a(l,2)ing#11,SET-a(p,1000000)rove#38,SET] size:4084896
47+
48+
layout filename=000005.sst
49+
----
50+
sstable
51+
├── data offset: 0 length: 46944
52+
├── data offset: 46949 length: 46945
53+
├── data offset: 93899 length: 46941
54+
├── data offset: 140845 length: 46944
55+
├── data offset: 187794 length: 46947
56+
├── data offset: 234746 length: 46945
57+
├── data offset: 281696 length: 46945
58+
├── data offset: 328646 length: 46943
59+
├── data offset: 375594 length: 46945
60+
├── data offset: 422544 length: 46940
61+
├── data offset: 469489 length: 46951
62+
├── data offset: 516445 length: 46941
63+
├── data offset: 563391 length: 46944
64+
├── data offset: 610340 length: 46945
65+
├── data offset: 657290 length: 46946
66+
├── data offset: 704241 length: 46940
67+
├── data offset: 751186 length: 46943
68+
├── data offset: 798134 length: 46942
69+
├── data offset: 845081 length: 46944
70+
├── data offset: 892030 length: 46945
71+
├── data offset: 938980 length: 46944
72+
├── data offset: 985929 length: 46945
73+
├── data offset: 1032879 length: 46945
74+
├── data offset: 1079829 length: 46945
75+
├── data offset: 1126779 length: 46946
76+
├── data offset: 1173730 length: 46947
77+
├── data offset: 1220682 length: 46943
78+
├── data offset: 1267630 length: 46944
79+
├── data offset: 1314579 length: 46942
80+
├── index offset: 1361526 length: 46942
81+
├── index offset: 1408473 length: 46946
82+
├── index offset: 1455424 length: 46942
83+
├── index offset: 1502371 length: 46945
84+
├── index offset: 1549321 length: 46948
85+
├── index offset: 1596274 length: 46946
86+
├── index offset: 1643225 length: 46946
87+
├── index offset: 1690176 length: 46944
88+
├── index offset: 1737125 length: 46946
89+
├── index offset: 1784076 length: 46941
90+
├── index offset: 1831022 length: 46952
91+
├── index offset: 1877979 length: 46942
92+
├── index offset: 1924926 length: 46945
93+
├── index offset: 1971876 length: 46946
94+
├── index offset: 2018827 length: 46947
95+
├── index offset: 2065779 length: 46941
96+
├── index offset: 2112725 length: 46944
97+
├── index offset: 2159674 length: 46943
98+
├── index offset: 2206622 length: 46945
99+
├── index offset: 2253572 length: 46946
100+
├── index offset: 2300523 length: 46945
101+
├── index offset: 2347473 length: 46946
102+
├── index offset: 2394424 length: 46946
103+
├── index offset: 2441375 length: 46946
104+
├── index offset: 2488326 length: 46947
105+
├── index offset: 2535278 length: 46948
106+
├── index offset: 2582231 length: 46944
107+
├── index offset: 2629180 length: 46945
108+
├── index offset: 2676130 length: 46943
109+
├── top-index offset: 2723078 length: 1361225
110+
├── properties offset: 4084308 length: 490
111+
├── meta-index offset: 4084803 length: 35
112+
└── footer offset: 4084843 length: 53
113+
114+
properties file=000005
115+
raw.key.size
116+
index.size
117+
index.partitions
118+
----
119+
raw.key.size:
120+
rocksdb.raw.key.size: 29000440
121+
index.size:
122+
rocksdb.index.size: 58001886
123+
rocksdb.top-level.index.size: 29000892
124+
index.partitions:
125+
rocksdb.index.partitions: 29
126+
127+
batch-commit
128+
del-range a(p,1000000)arition a(p,1000000)eal
129+
del-range a(p,1000000)ellate a(p,1000000)etizers
130+
del-range a(p,1000000)etizing a(p,1000000)lauding
131+
del-range a(p,1000000)le a(p,1000000)lebottomjeans
132+
del-range a(p,1000000)lebottomjeans a(p,1000000)lication
133+
del-range a(p,1000000)ly a(p,1000000)lying
134+
del-range a(p,1000000)raisals a(p,1000000)rentice
135+
del-range a(p,1000000)rentices a(p,1000000)roval
136+
del-range a(p,1000000)rovals a(p,1000000)rove
137+
----
138+
139+
flush verbose
140+
----
141+
L0.1:
142+
000008:[a(p,1000000)arition#39,RANGEDEL-a(p,1000000)rove#inf,RANGEDEL] seqnums:[39-47] points:[a(p,1000000)arition#39,RANGEDEL-a(p,1000000)rove#inf,RANGEDEL] size:18000932
143+
L0.0:
144+
000005:[a(p,1000000)a(l,2)ing#11,SET-a(p,1000000)rove#38,SET] seqnums:[10-38] points:[a(p,1000000)a(l,2)ing#11,SET-a(p,1000000)rove#38,SET] size:4084896
145+
146+
layout filename=000008.sst
147+
----
148+
sstable
149+
├── data offset: 0 length: 8
150+
├── index offset: 13 length: 24
151+
├── range-del offset: 42 length: 18000310
152+
├── properties offset: 18000357 length: 447
153+
├── meta-index offset: 18000809 length: 65
154+
└── footer offset: 18000879 length: 53
155+
156+
properties file=000008
157+
rocksdb.raw
158+
----
159+
rocksdb.raw:
160+
rocksdb.raw.key.size: 9000139
161+
rocksdb.raw.value.size: 9000068
162+
163+
# Repeat the above with columnar blocks.
164+
165+
define target-file-sizes=(4000000) format-major-version=19
166+
----
167+
168+
# Commit many keys with 1MB-shared prefixes.
169+
170+
batch-commit
171+
set a(p,1000000)arition
172+
set a(p,1000000)alling
173+
set a(p,1000000)eal
174+
set a(p,1000000)ellate
175+
set a(p,1000000)endectomy
176+
set a(p,1000000)etizers
177+
set a(p,1000000)etizing
178+
set a(p,1000000)laude
179+
set a(p,1000000)lauding
180+
set a(p,1000000)le
181+
set a(p,1000000)les
182+
set a(p,1000000)letini
183+
set a(p,1000000)letinis
184+
set a(p,1000000)lebottomjeans
185+
set a(p,1000000)lication
186+
set a(p,1000000)ly
187+
set a(p,1000000)lying
188+
set a(p,1000000)ollo
189+
set a(p,1000000)raisal
190+
set a(p,1000000)raisals
191+
set a(p,1000000)raiser
192+
set a(p,1000000)raisers
193+
set a(p,1000000)raising
194+
set a(p,1000000)rentice
195+
set a(p,1000000)rentices
196+
set a(p,1000000)renticing
197+
set a(p,1000000)roval
198+
set a(p,1000000)rovals
199+
set a(p,1000000)rove
200+
----
201+
202+
flush verbose
203+
----
204+
L0.0:
205+
000005:[a(p,1000000)a(l,2)ing#11,SET-a(p,1000000)e(l,2)ate#13,SET] seqnums:[10-13] points:[a(p,1000000)a(l,2)ing#11,SET-a(p,1000000)e(l,2)ate#13,SET] size:564287
206+
000006:[a(p,1000000)endectomy#14,SET-a(p,1000000)laude#17,SET] seqnums:[14-17] points:[a(p,1000000)endectomy#14,SET-a(p,1000000)laude#17,SET] size:564304
207+
000007:[a(p,1000000)lauding#18,SET-a(p,1000000)les#20,SET] seqnums:[18-23] points:[a(p,1000000)lauding#18,SET-a(p,1000000)les#20,SET] size:564298
208+
000008:[a(p,1000000)letini#21,SET-a(p,1000000)ly#25,SET] seqnums:[21-25] points:[a(p,1000000)letini#21,SET-a(p,1000000)ly#25,SET] size:564292
209+
000009:[a(p,1000000)lying#26,SET-a(p,1000000)raisals#29,SET] seqnums:[26-29] points:[a(p,1000000)lying#26,SET-a(p,1000000)raisals#29,SET] size:564286
210+
000010:[a(p,1000000)raiser#30,SET-a(p,1000000)rentice#33,SET] seqnums:[30-33] points:[a(p,1000000)raiser#30,SET-a(p,1000000)rentice#33,SET] size:564299
211+
000011:[a(p,1000000)rentices#34,SET-a(p,1000000)rovals#37,SET] seqnums:[34-37] points:[a(p,1000000)rentices#34,SET-a(p,1000000)rovals#37,SET] size:564311
212+
000012:[a(p,1000000)rove#38,SET-a(p,1000000)rove#38,SET] seqnums:[38-38] points:[a(p,1000000)rove#38,SET-a(p,1000000)rove#38,SET] size:94575
213+
214+
layout filename=000006.sst
215+
----
216+
sstable
217+
├── data offset: 0 length: 46998
218+
├── data offset: 47003 length: 46993
219+
├── data offset: 94001 length: 46993
220+
├── data offset: 140999 length: 46991
221+
├── index offset: 187995 length: 46967
222+
├── index offset: 234967 length: 46969
223+
├── index offset: 281941 length: 46973
224+
├── index offset: 328919 length: 46969
225+
├── top-index offset: 375893 length: 187752
226+
├── properties offset: 563650 length: 557
227+
├── meta-index offset: 564212 length: 34
228+
└── footer offset: 564251 length: 53
229+
230+
properties file=000006
231+
raw.key.size
232+
index.size
233+
index.partitions
234+
----
235+
raw.key.size:
236+
rocksdb.raw.key.size: 4000064
237+
index.size:
238+
rocksdb.index.size: 8000330
239+
rocksdb.top-level.index.size: 4000112
240+
index.partitions:
241+
rocksdb.index.partitions: 4
242+
243+
batch-commit
244+
del-range a(p,1000000)arition a(p,1000000)eal
245+
del-range a(p,1000000)ellate a(p,1000000)etizers
246+
del-range a(p,1000000)etizing a(p,1000000)lauding
247+
del-range a(p,1000000)le a(p,1000000)lebottomjeans
248+
del-range a(p,1000000)lebottomjeans a(p,1000000)lication
249+
del-range a(p,1000000)ly a(p,1000000)lying
250+
del-range a(p,1000000)raisals a(p,1000000)rentice
251+
del-range a(p,1000000)rentices a(p,1000000)roval
252+
del-range a(p,1000000)rovals a(p,1000000)rove
253+
----
254+
255+
flush verbose
256+
----
257+
L0.1:
258+
000015:[a(p,1000000)arition#39,RANGEDEL-a(p,1000000)lauding#inf,RANGEDEL] seqnums:[39-41] points:[a(p,1000000)arition#39,RANGEDEL-a(p,1000000)lauding#inf,RANGEDEL] size:6000807
259+
000016:[a(p,1000000)le#42,RANGEDEL-a(p,1000000)rentice#inf,RANGEDEL] seqnums:[42-45] points:[a(p,1000000)le#42,RANGEDEL-a(p,1000000)rentice#inf,RANGEDEL] size:7000821
260+
000017:[a(p,1000000)rentices#46,RANGEDEL-a(p,1000000)rove#inf,RANGEDEL] seqnums:[46-47] points:[a(p,1000000)rentices#46,RANGEDEL-a(p,1000000)rove#inf,RANGEDEL] size:4000779
261+
L0.0:
262+
000005:[a(p,1000000)a(l,2)ing#11,SET-a(p,1000000)e(l,2)ate#13,SET] seqnums:[10-13] points:[a(p,1000000)a(l,2)ing#11,SET-a(p,1000000)e(l,2)ate#13,SET] size:564287
263+
000006:[a(p,1000000)endectomy#14,SET-a(p,1000000)laude#17,SET] seqnums:[14-17] points:[a(p,1000000)endectomy#14,SET-a(p,1000000)laude#17,SET] size:564304
264+
000007:[a(p,1000000)lauding#18,SET-a(p,1000000)les#20,SET] seqnums:[18-23] points:[a(p,1000000)lauding#18,SET-a(p,1000000)les#20,SET] size:564298
265+
000008:[a(p,1000000)letini#21,SET-a(p,1000000)ly#25,SET] seqnums:[21-25] points:[a(p,1000000)letini#21,SET-a(p,1000000)ly#25,SET] size:564292
266+
000009:[a(p,1000000)lying#26,SET-a(p,1000000)raisals#29,SET] seqnums:[26-29] points:[a(p,1000000)lying#26,SET-a(p,1000000)raisals#29,SET] size:564286
267+
000010:[a(p,1000000)raiser#30,SET-a(p,1000000)rentice#33,SET] seqnums:[30-33] points:[a(p,1000000)raiser#30,SET-a(p,1000000)rentice#33,SET] size:564299
268+
000011:[a(p,1000000)rentices#34,SET-a(p,1000000)rovals#37,SET] seqnums:[34-37] points:[a(p,1000000)rentices#34,SET-a(p,1000000)rovals#37,SET] size:564311
269+
000012:[a(p,1000000)rove#38,SET-a(p,1000000)rove#38,SET] seqnums:[38-38] points:[a(p,1000000)rove#38,SET-a(p,1000000)rove#38,SET] size:94575
270+
271+
layout filename=000015.sst
272+
----
273+
sstable
274+
├── index offset: 0 length: 28
275+
├── range-del offset: 33 length: 6000129
276+
├── properties offset: 6000167 length: 512
277+
├── meta-index offset: 6000684 length: 65
278+
└── footer offset: 6000754 length: 53
279+
280+
properties file=000015
281+
rocksdb.raw
282+
----
283+
rocksdb.raw:
284+
rocksdb.raw.key.size: 6000043
285+
rocksdb.raw.value.size: 0

0 commit comments

Comments
 (0)