Skip to content
This repository was archived by the owner on Aug 13, 2019. It is now read-only.

Commit e95e806

Browse files
csmarchbankscsmarchbanks
committed
Better test coverage for compressed WAL
Many tests that use the WAL were only running in uncompressed mode, now appropriate tests will run in both modes. Signed-off-by: Chris Marchbanks <[email protected]>
1 parent f894ec0 commit e95e806

File tree

4 files changed

+426
-398
lines changed

4 files changed

+426
-398
lines changed

‎checkpoint_test.go‎

Lines changed: 95 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -86,108 +86,112 @@ func TestDeleteCheckpoints(t *testing.T) {
8686
}
8787

8888
func TestCheckpoint(t *testing.T) {
89-
dir, err := ioutil.TempDir("", "test_checkpoint")
90-
testutil.Ok(t, err)
91-
defer func() {
92-
testutil.Ok(t, os.RemoveAll(dir))
93-
}()
94-
95-
var enc RecordEncoder
96-
// Create a dummy segment to bump the initial number.
97-
seg, err := wal.CreateSegment(dir, 100)
98-
testutil.Ok(t, err)
99-
testutil.Ok(t, seg.Close())
100-
101-
// Manually create checkpoint for 99 and earlier.
102-
w, err := wal.New(nil, nil, filepath.Join(dir, "checkpoint.0099"), false)
103-
testutil.Ok(t, err)
104-
105-
// Add some data we expect to be around later.
106-
err = w.Log(enc.Series([]RefSeries{
107-
{Ref: 0, Labels: labels.FromStrings("a", "b", "c", "0")},
108-
{Ref: 1, Labels: labels.FromStrings("a", "b", "c", "1")},
109-
}, nil))
110-
testutil.Ok(t, err)
111-
testutil.Ok(t, w.Close())
112-
113-
// Start a WAL and write records to it as usual.
114-
w, err = wal.NewSize(nil, nil, dir, 64*1024, false)
115-
testutil.Ok(t, err)
89+
for _, compress := range []bool{false, true} {
90+
t.Run(fmt.Sprintf("compress=%t", compress), func(t *testing.T) {
91+
dir, err := ioutil.TempDir("", "test_checkpoint")
92+
testutil.Ok(t, err)
93+
defer func() {
94+
testutil.Ok(t, os.RemoveAll(dir))
95+
}()
11696

117-
var last int64
118-
for i := 0; ; i++ {
119-
_, n, err := w.Segments()
120-
testutil.Ok(t, err)
121-
if n >= 106 {
122-
break
123-
}
124-
// Write some series initially.
125-
if i == 0 {
126-
b := enc.Series([]RefSeries{
127-
{Ref: 2, Labels: labels.FromStrings("a", "b", "c", "2")},
128-
{Ref: 3, Labels: labels.FromStrings("a", "b", "c", "3")},
129-
{Ref: 4, Labels: labels.FromStrings("a", "b", "c", "4")},
130-
{Ref: 5, Labels: labels.FromStrings("a", "b", "c", "5")},
131-
}, nil)
132-
testutil.Ok(t, w.Log(b))
133-
}
134-
// Write samples until the WAL has enough segments.
135-
// Make them have drifting timestamps within a record to see that they
136-
// get filtered properly.
137-
b := enc.Samples([]RefSample{
138-
{Ref: 0, T: last, V: float64(i)},
139-
{Ref: 1, T: last + 10000, V: float64(i)},
140-
{Ref: 2, T: last + 20000, V: float64(i)},
141-
{Ref: 3, T: last + 30000, V: float64(i)},
142-
}, nil)
143-
testutil.Ok(t, w.Log(b))
144-
145-
last += 100
146-
}
147-
testutil.Ok(t, w.Close())
97+
var enc RecordEncoder
98+
// Create a dummy segment to bump the initial number.
99+
seg, err := wal.CreateSegment(dir, 100)
100+
testutil.Ok(t, err)
101+
testutil.Ok(t, seg.Close())
148102

149-
_, err = Checkpoint(w, 100, 106, func(x uint64) bool {
150-
return x%2 == 0
151-
}, last/2)
152-
testutil.Ok(t, err)
153-
testutil.Ok(t, w.Truncate(107))
154-
testutil.Ok(t, DeleteCheckpoints(w.Dir(), 106))
103+
// Manually create checkpoint for 99 and earlier.
104+
w, err := wal.New(nil, nil, filepath.Join(dir, "checkpoint.0099"), compress)
105+
testutil.Ok(t, err)
155106

156-
// Only the new checkpoint should be left.
157-
files, err := fileutil.ReadDir(dir)
158-
testutil.Ok(t, err)
159-
testutil.Equals(t, 1, len(files))
160-
testutil.Equals(t, "checkpoint.000106", files[0])
107+
// Add some data we expect to be around later.
108+
err = w.Log(enc.Series([]RefSeries{
109+
{Ref: 0, Labels: labels.FromStrings("a", "b", "c", "0")},
110+
{Ref: 1, Labels: labels.FromStrings("a", "b", "c", "1")},
111+
}, nil))
112+
testutil.Ok(t, err)
113+
testutil.Ok(t, w.Close())
161114

162-
sr, err := wal.NewSegmentsReader(filepath.Join(dir, "checkpoint.000106"))
163-
testutil.Ok(t, err)
164-
defer sr.Close()
115+
// Start a WAL and write records to it as usual.
116+
w, err = wal.NewSize(nil, nil, dir, 64*1024, compress)
117+
testutil.Ok(t, err)
165118

166-
var dec RecordDecoder
167-
var series []RefSeries
168-
r := wal.NewReader(sr)
119+
var last int64
120+
for i := 0; ; i++ {
121+
_, n, err := w.Segments()
122+
testutil.Ok(t, err)
123+
if n >= 106 {
124+
break
125+
}
126+
// Write some series initially.
127+
if i == 0 {
128+
b := enc.Series([]RefSeries{
129+
{Ref: 2, Labels: labels.FromStrings("a", "b", "c", "2")},
130+
{Ref: 3, Labels: labels.FromStrings("a", "b", "c", "3")},
131+
{Ref: 4, Labels: labels.FromStrings("a", "b", "c", "4")},
132+
{Ref: 5, Labels: labels.FromStrings("a", "b", "c", "5")},
133+
}, nil)
134+
testutil.Ok(t, w.Log(b))
135+
}
136+
// Write samples until the WAL has enough segments.
137+
// Make them have drifting timestamps within a record to see that they
138+
// get filtered properly.
139+
b := enc.Samples([]RefSample{
140+
{Ref: 0, T: last, V: float64(i)},
141+
{Ref: 1, T: last + 10000, V: float64(i)},
142+
{Ref: 2, T: last + 20000, V: float64(i)},
143+
{Ref: 3, T: last + 30000, V: float64(i)},
144+
}, nil)
145+
testutil.Ok(t, w.Log(b))
146+
147+
last += 100
148+
}
149+
testutil.Ok(t, w.Close())
169150

170-
for r.Next() {
171-
rec := r.Record()
151+
_, err = Checkpoint(w, 100, 106, func(x uint64) bool {
152+
return x%2 == 0
153+
}, last/2)
154+
testutil.Ok(t, err)
155+
testutil.Ok(t, w.Truncate(107))
156+
testutil.Ok(t, DeleteCheckpoints(w.Dir(), 106))
172157

173-
switch dec.Type(rec) {
174-
case RecordSeries:
175-
series, err = dec.Series(rec, series)
158+
// Only the new checkpoint should be left.
159+
files, err := fileutil.ReadDir(dir)
176160
testutil.Ok(t, err)
177-
case RecordSamples:
178-
samples, err := dec.Samples(rec, nil)
161+
testutil.Equals(t, 1, len(files))
162+
testutil.Equals(t, "checkpoint.000106", files[0])
163+
164+
sr, err := wal.NewSegmentsReader(filepath.Join(dir, "checkpoint.000106"))
179165
testutil.Ok(t, err)
180-
for _, s := range samples {
181-
testutil.Assert(t, s.T >= last/2, "sample with wrong timestamp")
166+
defer sr.Close()
167+
168+
var dec RecordDecoder
169+
var series []RefSeries
170+
r := wal.NewReader(sr)
171+
172+
for r.Next() {
173+
rec := r.Record()
174+
175+
switch dec.Type(rec) {
176+
case RecordSeries:
177+
series, err = dec.Series(rec, series)
178+
testutil.Ok(t, err)
179+
case RecordSamples:
180+
samples, err := dec.Samples(rec, nil)
181+
testutil.Ok(t, err)
182+
for _, s := range samples {
183+
testutil.Assert(t, s.T >= last/2, "sample with wrong timestamp")
184+
}
185+
}
182186
}
183-
}
187+
testutil.Ok(t, r.Err())
188+
testutil.Equals(t, []RefSeries{
189+
{Ref: 0, Labels: labels.FromStrings("a", "b", "c", "0")},
190+
{Ref: 2, Labels: labels.FromStrings("a", "b", "c", "2")},
191+
{Ref: 4, Labels: labels.FromStrings("a", "b", "c", "4")},
192+
}, series)
193+
})
184194
}
185-
testutil.Ok(t, r.Err())
186-
testutil.Equals(t, []RefSeries{
187-
{Ref: 0, Labels: labels.FromStrings("a", "b", "c", "0")},
188-
{Ref: 2, Labels: labels.FromStrings("a", "b", "c", "2")},
189-
{Ref: 4, Labels: labels.FromStrings("a", "b", "c", "4")},
190-
}, series)
191195
}
192196

193197
func TestCheckpointNoTmpFolderAfterError(t *testing.T) {

0 commit comments

Comments
 (0)