Skip to content
This repository was archived by the owner on Aug 13, 2019. It is now read-only.

Commit ce55654

Browse files
naivewongnaivewong
committed
add new baseDeltaPostings
Signed-off-by: naivewong <[email protected]>
1 parent ea7b5b5 commit ce55654

File tree

3 files changed

+76
-48
lines changed

3 files changed

+76
-48
lines changed

‎index/index.go‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -536,10 +536,15 @@ func (w *Writer) WritePostings(name, value string, it Postings) error {
536536
// The base.
537537
w.buf2.PutUvarint32(refs[0])
538538
// The width.
539-
width := bits.Len32(uint32(refs[len(refs)-1] - refs[0]))
539+
width := (bits.Len32(refs[len(refs)-1] - refs[0]) + 7) >> 3
540+
if width == 0 {
541+
width = 1
542+
}
540543
w.buf2.PutByte(byte(width))
541-
for _, r := range refs {
542-
w.buf2.PutBits(uint64(r-refs[0]), width)
544+
for i := 0; i < len(refs); i++ {
545+
for j := width - 1; j >= 0; j-- {
546+
w.buf2.B = append(w.buf2.B, byte(((refs[i]-refs[0])>>(8*uint(j))&0xff)))
547+
}
543548
}
544549
case 3:
545550
writeDeltaBlockPostings(&w.buf2, refs)

‎index/postings.go‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -693,7 +693,7 @@ func (it *bigEndianPostings) Err() error {
693693
}
694694

695695
// 1 is bigEndian, 2 is baseDelta, 3 is deltaBlock, 4 is baseDeltaBlock, 5 is bitmapPostings, 6 is roaringBitmapPostings.
696-
const postingsType = 6
696+
const postingsType = 2
697697

698698
type bitSlice struct {
699699
bstream []byte
@@ -747,25 +747,41 @@ func (bs *bitSlice) readBits(offset int) uint64 {
747747
// │ num <4b> │ base <uvarint> │ width <1b> │ delta 1 <bits> │ ... │ delta n <bits> │
748748
// └──────────┴────────────────┴────────────┴────────────────┴─────┴────────────────┘
749749
type baseDeltaPostings struct {
750-
bs bitSlice
751-
base uint32
752-
size int
753-
idx int
754-
cur uint64
750+
bs []byte
751+
width int
752+
base uint32
753+
size int
754+
idx int
755+
cur uint64
756+
mask uint32
757+
prel int
755758
}
756759

757760
func newBaseDeltaPostings(bstream []byte, base uint32, width int, size int) *baseDeltaPostings {
758-
return &baseDeltaPostings{bs: bitSlice{bstream: bstream, width: width}, base: base, size: size, cur: uint64(base)}
761+
return &baseDeltaPostings{bs: bstream, width: width, base: base, size: size, cur: uint64(base), mask: (uint32(1) << (uint32(width)<<3)) - 1, prel: 4 - width}
762+
}
763+
764+
func (it *baseDeltaPostings) readBytes(off int) int {
765+
val := 0
766+
for i := 0; i < it.width; i ++ {
767+
val = (val << 8) | int(it.bs[off+i])
768+
}
769+
return val
759770
}
760771

761772
func (it *baseDeltaPostings) At() uint64 {
762773
return it.cur
763774
}
764775

765776
func (it *baseDeltaPostings) Next() bool {
766-
if it.size > it.idx {
767-
it.cur = it.bs.readBits(it.idx*it.bs.width) + uint64(it.base)
768-
it.idx += 1
777+
if it.idx < it.size*it.width {
778+
if it.idx-it.prel < 0 {
779+
it.cur = uint64(it.readBytes(it.idx)) + uint64(it.base)
780+
it.idx += it.width
781+
return true
782+
}
783+
it.cur = uint64(binary.BigEndian.Uint32(it.bs[it.idx-it.prel:])&it.mask) + uint64(it.base)
784+
it.idx += it.width
769785
return true
770786
}
771787
return false
@@ -776,18 +792,19 @@ func (it *baseDeltaPostings) Seek(x uint64) bool {
776792
return true
777793
}
778794

779-
num := it.size - it.idx
795+
num := it.size - it.idx / it.width
780796
// Do binary search between current position and end.
781797
x -= uint64(it.base)
782798
i := sort.Search(num, func(i int) bool {
783-
return it.bs.readBits((i+it.idx)*it.bs.width) >= x
799+
return uint64(binary.BigEndian.Uint32(it.bs[it.idx+i*it.width-it.prel:])&it.mask) >= x
784800
})
785801
if i < num {
786-
it.cur = it.bs.readBits((i+it.idx)*it.bs.width) + uint64(it.base)
787-
it.idx += i
802+
it.idx += i*it.width
803+
it.cur = uint64(it.base) + uint64(binary.BigEndian.Uint32(it.bs[it.idx-it.prel:])&it.mask)
804+
it.idx += it.width
788805
return true
789806
}
790-
it.idx += i
807+
it.idx += i*it.width
791808
return false
792809
}
793810

‎index/postings_test.go‎

Lines changed: 36 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -729,10 +729,12 @@ func TestBaseDeltaPostings(t *testing.T) {
729729
ls[i] = ls[i-1] + uint32(rand.Int31n(25)) + 2
730730
}
731731

732-
width := bits.Len32(ls[len(ls)-1] - ls[0])
732+
width := (bits.Len32(ls[len(ls)-1] - ls[0]) + 7) >> 3
733733
buf := encoding.Encbuf{}
734734
for i := 0; i < num; i++ {
735-
buf.PutBits(uint64(ls[i]-ls[0]), width)
735+
for j := width - 1; j >= 0; j-- {
736+
buf.B = append(buf.B, byte(((ls[i]-ls[0])>>(8*uint(j))&0xff)))
737+
}
736738
}
737739
// t.Log("(baseDeltaPostings) len of 1000 number = ", len(buf.Get()))
738740

@@ -1100,7 +1102,8 @@ func BenchmarkPostings(b *testing.B) {
11001102
ls := make([]uint32, num)
11011103
ls[0] = 2
11021104
for i := 1; i < num; i++ {
1103-
ls[i] = ls[i-1] + uint32(rand.Int31n(15)) + 2
1105+
ls[i] = ls[i-1] + uint32(rand.Int31n(25)) + 2
1106+
// ls[i] = ls[i-1] + 2
11041107
}
11051108

11061109
// bigEndianPostings.
@@ -1112,10 +1115,13 @@ func BenchmarkPostings(b *testing.B) {
11121115
b.Log("bigEndianPostings size =", len(bufBE))
11131116

11141117
// baseDeltaPostings.
1115-
width := bits.Len32(ls[len(ls)-1] - ls[0])
1118+
width := (bits.Len32(ls[len(ls)-1] - ls[0]) + 7) >> 3
11161119
bufBD := encoding.Encbuf{}
11171120
for i := 0; i < num; i++ {
1118-
bufBD.PutBits(uint64(ls[i]-ls[0]), width)
1121+
for j := width - 1; j >= 0; j-- {
1122+
bufBD.B = append(bufBD.B, byte(((ls[i]-ls[0])>>(8*uint(j))&0xff)))
1123+
}
1124+
// bufBD.PutBits(uint64(ls[i]-ls[0]), width)
11191125
}
11201126
b.Log("baseDeltaPostings size =", len(bufBD.Get()))
11211127

@@ -1232,20 +1238,20 @@ func BenchmarkPostings(b *testing.B) {
12321238
testutil.Assert(bench, bep.Err() == nil, "")
12331239
}
12341240
})
1235-
// b.Run("baseDeltaIteration", func(bench *testing.B) {
1236-
// bench.ResetTimer()
1237-
// bench.ReportAllocs()
1238-
// for j := 0; j < bench.N; j++ {
1239-
// bdp := newBaseDeltaPostings(bufBD.Get(), ls[0], width, len(ls))
1241+
b.Run("baseDeltaIteration", func(bench *testing.B) {
1242+
bench.ResetTimer()
1243+
bench.ReportAllocs()
1244+
for j := 0; j < bench.N; j++ {
1245+
bdp := newBaseDeltaPostings(bufBD.Get(), ls[0], width, len(ls))
12401246

1241-
// for i := 0; i < num; i++ {
1242-
// testutil.Assert(bench, bdp.Next() == true, "")
1243-
// testutil.Equals(bench, uint64(ls[i]), bdp.At())
1244-
// }
1245-
// testutil.Assert(bench, bdp.Next() == false, "")
1246-
// testutil.Assert(bench, bdp.Err() == nil, "")
1247-
// }
1248-
// })
1247+
for i := 0; i < num; i++ {
1248+
testutil.Assert(bench, bdp.Next() == true, "")
1249+
testutil.Equals(bench, uint64(ls[i]), bdp.At())
1250+
}
1251+
testutil.Assert(bench, bdp.Next() == false, "")
1252+
testutil.Assert(bench, bdp.Err() == nil, "")
1253+
}
1254+
})
12491255
// b.Run("deltaBlockIteration", func(bench *testing.B) {
12501256
// bench.ResetTimer()
12511257
// bench.ReportAllocs()
@@ -1316,19 +1322,19 @@ func BenchmarkPostings(b *testing.B) {
13161322
}
13171323
}
13181324
})
1319-
// b.Run("baseDeltaSeek", func(bench *testing.B) {
1320-
// bench.ResetTimer()
1321-
// bench.ReportAllocs()
1322-
// for j := 0; j < bench.N; j++ {
1323-
// bdp := newBaseDeltaPostings(bufBD.Get(), ls[0], width, len(ls))
1325+
b.Run("baseDeltaSeek", func(bench *testing.B) {
1326+
bench.ResetTimer()
1327+
bench.ReportAllocs()
1328+
for j := 0; j < bench.N; j++ {
1329+
bdp := newBaseDeltaPostings(bufBD.Get(), ls[0], width, len(ls))
13241330

1325-
// for _, v := range table {
1326-
// testutil.Equals(bench, v.found, bdp.Seek(uint64(v.seek)))
1327-
// testutil.Equals(bench, uint64(v.val), bdp.At())
1328-
// testutil.Assert(bench, bdp.Err() == nil, "")
1329-
// }
1330-
// }
1331-
// })
1331+
for _, v := range table {
1332+
testutil.Equals(bench, v.found, bdp.Seek(uint64(v.seek)))
1333+
testutil.Equals(bench, uint64(v.val), bdp.At())
1334+
testutil.Assert(bench, bdp.Err() == nil, "")
1335+
}
1336+
}
1337+
})
13321338
// b.Run("deltaBlockSeek", func(bench *testing.B) {
13331339
// bench.ResetTimer()
13341340
// bench.ReportAllocs()

0 commit comments

Comments
 (0)