Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reducing size #8

Merged
merged 25 commits into from
May 26, 2022
Merged
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
b6b993d
feat: add trunk compress for storedFields
hengfeiyang Apr 28, 2022
7f1692a
feat: use zstd replace snappy
hengfeiyang Apr 28, 2022
a5cb202
feat: compress docValue
hengfeiyang May 1, 2022
c3633ed
feat: reback the docValue compress
hengfeiyang May 1, 2022
4662783
feat: packed docNum and Offset for docValue
hengfeiyang May 1, 2022
439e31a
doc: update go.mod
hengfeiyang May 1, 2022
0753dc9
feat: packed numeric of posting list
hengfeiyang May 1, 2022
3061032
feat: compress numeric of posting list
hengfeiyang May 1, 2022
b4acb45
feat: packed numeric of posting list
hengfeiyang May 2, 2022
3442f5a
feat: compress intcoder
hengfeiyang May 2, 2022
152a011
feat: run optimize on bitmap
hengfeiyang May 3, 2022
c8cc4b7
feat: optimize document values chunk
hengfeiyang May 3, 2022
79196b6
doc: add author
hengfeiyang May 3, 2022
9bed194
style: change implement
hengfeiyang May 3, 2022
4f5e6c0
fix: tests
hengfeiyang May 3, 2022
929ccaf
test: add test for document coder
hengfeiyang May 3, 2022
ace50a7
style: format code
hengfeiyang May 4, 2022
dd60c36
update trunk to chunk
hengfeiyang May 15, 2022
ab3514f
update sort of Authors
hengfeiyang May 15, 2022
2f4d2fd
rename BufferSize to Size and remove Close method
hengfeiyang May 15, 2022
5e1f80d
update version
hengfeiyang May 15, 2022
d802dbc
fix panic when search memory
hengfeiyang May 24, 2022
1e2159e
rename variables
hengfeiyang May 25, 2022
c784da3
Merge branch 'master'
hengfeiyang May 26, 2022
a14a43c
Merge branch 'blugelabs-master'
hengfeiyang May 26, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rename BufferSize to Size and remove Close method
  • Loading branch information
hengfeiyang committed May 15, 2022
commit 2f4d2fdf0e8892e01a944ad01625646f55fe0e0c
14 changes: 6 additions & 8 deletions documentcoder.go
Original file line number Diff line number Diff line change
@@ -86,6 +86,10 @@ func (c *chunkedDocumentCoder) flush() error {
}

func (c *chunkedDocumentCoder) Write() error {
// flush first
if err := c.flush(); err != nil {
return err
}
var err error
var wn, n int
// write chunk offsets
@@ -109,12 +113,6 @@ func (c *chunkedDocumentCoder) Write() error {
return nil
}

// Close indicates you are done calling Add() this allows
// the final chunk to be encoded.
func (c *chunkedDocumentCoder) Close() error {
return c.flush()
}

func (c *chunkedDocumentCoder) Reset() {
c.compressed = c.compressed[:0]
c.offsets = c.offsets[:0]
@@ -123,8 +121,8 @@ func (c *chunkedDocumentCoder) Reset() {
c.buf.Reset()
}

// BufferSize returns buffer len
func (c *chunkedDocumentCoder) BufferSize() uint64 {
// Size returns buffer size of current chunk
func (c *chunkedDocumentCoder) Size() uint64 {
return uint64(c.buf.Len())
}

3 changes: 0 additions & 3 deletions documentcoder_test.go
Original file line number Diff line number Diff line change
@@ -52,7 +52,6 @@ func TestChunkedDocumentCoder(t *testing.T) {
t.Fatalf("error adding to documentcoder: %v", err)
}
}
_ = cic.Close()
err := cic.Write()
if err != nil {
t.Fatalf("error writing: %v", err)
@@ -103,8 +102,6 @@ func TestChunkedDocumentCoders(t *testing.T) {
t.Fatalf("error adding to documentcoder: %v", err)
}
}
_ = cic1.Close()
_ = cic2.Close()

err := cic1.Write()
if err != nil {
7 changes: 2 additions & 5 deletions merge.go
Original file line number Diff line number Diff line change
@@ -682,9 +682,6 @@ func mergeStoredAndRemap(segments []*Segment, drops []*roaring.Bitmap,
}

// document chunk coder
if err := docChunkCoder.Close(); err != nil {
return 0, nil, err
}
if err := docChunkCoder.Write(); err != nil {
return 0, nil, err
}
@@ -749,7 +746,7 @@ func mergeStoredAndRemapSegment(seg *Segment, dropsI *roaring.Bitmap, segNewDocN
metaBytes := metaBuf.Bytes()

// record where we're about to start writing
docNumOffsets[newDocNum] = docChunkCoder.BufferSize()
docNumOffsets[newDocNum] = docChunkCoder.Size()
// document chunk line
if _, err := docChunkCoder.Add(newDocNum, metaBytes, data); err != nil {
return 0, err
@@ -795,7 +792,7 @@ func (s *Segment) copyStoredDocs(newDocNum uint64, newDocNumOffsets []uint64, do
dataLenData := uncompressed[storedOffset+n : storedOffset+n+int(binary.MaxVarintLen64)]
dataLen, read := binary.Uvarint(dataLenData)
n += read
newDocNumOffsets[newDocNum] = docChunkCoder.BufferSize()
newDocNumOffsets[newDocNum] = docChunkCoder.Size()
metaBytes := uncompressed[storedOffset+n : storedOffset+n+int(metaLen)]
data := uncompressed[storedOffset+n+int(metaLen) : storedOffset+n+int(metaLen)+int(dataLen)]
if _, err := docChunkCoder.Add(newDocNum, metaBytes, data); err != nil {
6 changes: 1 addition & 5 deletions new.go
Original file line number Diff line number Diff line change
@@ -609,18 +609,14 @@ func (s *interim) writeStoredFields() (
}

metaBytes := s.metaBuf.Bytes()
docStoredOffsets[docNum] = docChunkCoder.BufferSize()
docStoredOffsets[docNum] = docChunkCoder.Size()
_, err = docChunkCoder.Add(uint64(docNum), metaBytes, data)
if err != nil {
return 0, err
}
}

// document chunk coder
err = docChunkCoder.Close()
if err != nil {
return 0, err
}
err = docChunkCoder.Write()
if err != nil {
return 0, err