Skip to content

Commit

Permalink
fix: progress data races (#17)
Browse files Browse the repository at this point in the history
Packfile writers are created and the index gets built in a separate
goroutine. A data race occurred when setting up the progress
collector. I plumbed the progress collector into the `newPackWrite`
funtion outright. That way, the call to `buildIndex` could take
it. Furthermore, using a mutex to serialize reporting for receiving
objects and resolving deltas.
  • Loading branch information
John Nelson authored Mar 31, 2020
1 parent ea26bcf commit 898b40e
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 14 deletions.
16 changes: 16 additions & 0 deletions plumbing/format/packfile/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"io"
"io/ioutil"
"sync"

"github.com/goabstract/go-git/v5/plumbing"
"github.com/goabstract/go-git/v5/plumbing/cache"
Expand Down Expand Up @@ -55,6 +56,7 @@ type Parser struct {

ob []Observer

m sync.Mutex
ProgressCollector *progress.Collector
objectsSeen uint32
deltasTotal uint32
Expand Down Expand Up @@ -94,7 +96,18 @@ func NewParserWithStorage(
}, nil
}

// UseProgressCollector sets up the parser to use a *progress.Collector
func (p *Parser) UseProgressCollector(c *progress.Collector) {
p.m.Lock()
defer p.m.Unlock()

p.ProgressCollector = c
}

func (p *Parser) writeDeltaProgress() {
p.m.Lock()
defer p.m.Unlock()

if p.ProgressCollector == nil {
return
}
Expand All @@ -103,6 +116,9 @@ func (p *Parser) writeDeltaProgress() {
}

func (p *Parser) writeObjectProgress() {
p.m.Lock()
defer p.m.Unlock()

if p.ProgressCollector == nil {
return
}
Expand Down
5 changes: 3 additions & 2 deletions storage/filesystem/dotgit/dotgit.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/go-git/go-billy/v5/osfs"
"github.com/goabstract/go-git/v5/plumbing"
"github.com/goabstract/go-git/v5/plumbing/progress"
"github.com/goabstract/go-git/v5/storage"
"github.com/goabstract/go-git/v5/utils/ioutil"

Expand Down Expand Up @@ -193,9 +194,9 @@ func (d *DotGit) Shallow() (billy.File, error) {

// NewObjectPack return a writer for a new packfile, it saves the packfile to
// disk and also generates and save the index for the given packfile.
func (d *DotGit) NewObjectPack() (*PackWriter, error) {
func (d *DotGit) NewObjectPack(pc *progress.Collector) (*PackWriter, error) {
d.cleanPackList()
return newPackWrite(d.fs)
return newPackWrite(d.fs, pc)
}

// ObjectPacks returns the list of availables packfiles
Expand Down
12 changes: 6 additions & 6 deletions storage/filesystem/dotgit/writers.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ import (
// is renamed/moved (depends on the Filesystem implementation) to the final
// location, if the PackWriter is not used, nothing is written
type PackWriter struct {
Notify func(plumbing.Hash, *idxfile.Writer)
ProgressCollector *progress.Collector
Notify func(plumbing.Hash, *idxfile.Writer)

fs billy.Filesystem
fr, fw billy.File
Expand All @@ -33,7 +32,7 @@ type PackWriter struct {
result chan error
}

func newPackWrite(fs billy.Filesystem) (*PackWriter, error) {
func newPackWrite(fs billy.Filesystem, pc *progress.Collector) (*PackWriter, error) {
fw, err := fs.TempFile(fs.Join(objectsPath, packPath), "tmp_pack_")
if err != nil {
return nil, err
Expand All @@ -52,21 +51,22 @@ func newPackWrite(fs billy.Filesystem) (*PackWriter, error) {
result: make(chan error),
}

go writer.buildIndex()
go writer.buildIndex(pc)
return writer, nil
}

func (w *PackWriter) buildIndex() {
func (w *PackWriter) buildIndex(pc *progress.Collector) {
s := packfile.NewScanner(w.synced)
w.writer = new(idxfile.Writer)
var err error
w.parser, err = packfile.NewParser(s, w.writer)
w.parser.ProgressCollector = w.ProgressCollector
if err != nil {
w.result <- err
return
}

w.parser.UseProgressCollector(pc)

checksum, err := w.parser.Parse()
if err != nil {
w.result <- err
Expand Down
6 changes: 3 additions & 3 deletions storage/filesystem/dotgit/writers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func (s *SuiteDotGit) TestNewObjectPack(c *C) {
fs := osfs.New(dir)
dot := New(fs)

w, err := dot.NewObjectPack()
w, err := dot.NewObjectPack(nil)
c.Assert(err, IsNil)

_, err = io.Copy(w, f.Packfile())
Expand Down Expand Up @@ -75,7 +75,7 @@ func (s *SuiteDotGit) TestNewObjectPackUnused(c *C) {
fs := osfs.New(dir)
dot := New(fs)

w, err := dot.NewObjectPack()
w, err := dot.NewObjectPack(nil)
c.Assert(err, IsNil)

c.Assert(w.Close(), IsNil)
Expand Down Expand Up @@ -146,7 +146,7 @@ func (s *SuiteDotGit) TestPackWriterUnusedNotify(c *C) {

fs := osfs.New(dir)

w, err := newPackWrite(fs)
w, err := newPackWrite(fs, nil)
c.Assert(err, IsNil)

w.Notify = func(h plumbing.Hash, idx *idxfile.Writer) {
Expand Down
4 changes: 1 addition & 3 deletions storage/filesystem/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,13 +99,11 @@ func (s *ObjectStorage) packfileWriter(pc *progress.Collector) (io.WriteCloser,
return nil, err
}

w, err := s.dir.NewObjectPack()
w, err := s.dir.NewObjectPack(pc)
if err != nil {
return nil, err
}

w.ProgressCollector = pc

w.Notify = func(h plumbing.Hash, writer *idxfile.Writer) {
index, err := writer.Index()
if err == nil {
Expand Down

0 comments on commit 898b40e

Please sign in to comment.