diff --git a/evaluate/metrics.go b/evaluate/metrics.go index 6f12a66e..3a5e2da9 100644 --- a/evaluate/metrics.go +++ b/evaluate/metrics.go @@ -7,6 +7,7 @@ import ( "strings" "golang.org/x/exp/maps" + "gonum.org/v1/gonum/stat" ) // Metrics holds numerical benchmarking metrics. @@ -15,6 +16,9 @@ type Metrics struct { Total uint // Executed is the number of benchmarking candidates with successful execution. Executed uint + + // Coverage holds the coverage of the benchmarking candidates. + Coverage []float64 } // Add sums two metrics objects. @@ -22,17 +26,19 @@ func (m Metrics) Add(o Metrics) Metrics { return Metrics{ Total: m.Total + o.Total, Executed: m.Executed + o.Total, + + Coverage: append(m.Coverage, o.Coverage...), } } // String returns a string representation of the metrics. func (m Metrics) String() string { - return fmt.Sprintf("#executed=%d/%d", m.Executed, m.Total) + return fmt.Sprintf("#executed=%d/%d, coverage=%v", m.Executed, m.Total, m.Coverage) } // String returns a string representation of the metrics. func (m Metrics) StringPercentage() string { - return fmt.Sprintf("#executed=%3.0f%%", float64(m.Executed)/float64(m.Total)*100.0) + return fmt.Sprintf("#executed=%3.0f%%, average coverage=%3.0f%%", float64(m.Executed)/float64(m.Total)*100.0, stat.Mean(m.Coverage, nil)) } // StringCSV returns a CSV row string representation of the metrics. @@ -40,6 +46,7 @@ func (m Metrics) StringCSV() []string { return []string{ fmt.Sprintf("%d", m.Total), fmt.Sprintf("%d", m.Executed), + fmt.Sprintf("%.0f", stat.Mean(m.Coverage, nil)), } } diff --git a/evaluate/metrics_test.go b/evaluate/metrics_test.go index a6fca274..472105fa 100644 --- a/evaluate/metrics_test.go +++ b/evaluate/metrics_test.go @@ -33,7 +33,7 @@ func TestFormatStringCSV(t *testing.T) { }, ExpectedString: ` - Category,0,0 + Category,0,0,NaN `, }) validate(t, &testCase{ @@ -43,16 +43,18 @@ func TestFormatStringCSV(t *testing.T) { "CategoryA": Metrics{ Total: 5, Executed: 3, + Coverage: []float64{100.0}, }, "CategoryB": Metrics{ Total: 4, Executed: 2, + Coverage: []float64{70.0}, }, }, ExpectedString: ` - CategoryA,5,3 - CategoryB,4,2 + CategoryA,5,3,100 + CategoryB,4,2,70 `, }) } diff --git a/evaluate/repository.go b/evaluate/repository.go index d91006e3..ebd1b32a 100644 --- a/evaluate/repository.go +++ b/evaluate/repository.go @@ -3,6 +3,7 @@ package evaluate import ( "errors" "log" + "math" "os" "path/filepath" @@ -51,12 +52,16 @@ func EvaluateRepository(model model.Model, language language.Language, repositor continue } - if err := language.Execute(temporaryRepositoryPath); err != nil { + coverage, err := language.Execute(temporaryRepositoryPath) + if err != nil { problems = append(problems, pkgerrors.WithMessage(err, filePath)) continue } metrics.Total++ + if !math.IsNaN(coverage) { + metrics.Coverage = append(metrics.Coverage, coverage) + } } return metrics, problems, nil diff --git a/go.mod b/go.mod index 5585f313..5f89ea04 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/zimmski/osutil v1.1.0 golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 + gonum.org/v1/gonum v0.15.0 ) require ( diff --git a/go.sum b/go.sum index 567cec7f..96752079 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/language/golang.go b/language/golang.go index d5c54655..77b62901 100644 --- a/language/golang.go +++ b/language/golang.go @@ -2,6 +2,7 @@ package language import ( "errors" + "fmt" "os" "path/filepath" "regexp" @@ -53,15 +54,18 @@ func (language *LanguageGolang) Files(repositoryPath string) (filePaths []string } var languageGoNoTestsMatch = regexp.MustCompile(`(?m)^DONE (\d+) tests.*in (.+?)$`) +var languageGoCoverageMatch = regexp.MustCompile(`(?m)^coverage: (\d+\.?\d+)% of statements`) +var languageGoNoCoverageMatch = regexp.MustCompile(`(?m)^coverage: \[no statements\]$`) // Execute invokes the language specific testing on the given repository. -func (language *LanguageGolang) Execute(repositoryPath string) (err error) { +func (language *LanguageGolang) Execute(repositoryPath string) (coverage float64, err error) { stdout, _, err := util.CommandWithResult(&util.Command{ Command: []string{ "gotestsum", "--format", "standard-verbose", // Keep formatting consistent. "--hide-summary", "skipped", // We are not interested in skipped tests, because they are the same as no tests at all. "--", // Let the real Go "test" tool options begin. + "-cover", // Enable statement coverage. "-v", // Output with the maximum information for easier debugging. "-vet=off", // Disable all linter checks, because those should be part of a different task. "./...", // Always execute all tests of the repository in case multiple test files have been generated. @@ -70,19 +74,33 @@ func (language *LanguageGolang) Execute(repositoryPath string) (err error) { Directory: repositoryPath, }) if err != nil { - return pkgerrors.WithStack(err) + return 0.0, pkgerrors.WithStack(err) } ms := languageGoNoTestsMatch.FindStringSubmatch(stdout) if ms == nil { - return pkgerrors.WithStack(errors.New("could not find Go test summary")) + return 0.0, pkgerrors.WithStack(errors.New("could not find Go test summary")) } testCount, err := strconv.ParseUint(ms[1], 10, 64) if err != nil { - return pkgerrors.WithStack(err) + return 0.0, pkgerrors.WithStack(err) } else if testCount == 0 { - return pkgerrors.WithStack(ErrNoTestFound) + return 0.0, pkgerrors.WithStack(ErrNoTestFound) } - return nil + if languageGoNoCoverageMatch.MatchString(stdout) { + return 0.0, nil + } + + mc := languageGoCoverageMatch.FindStringSubmatch(stdout) + if mc == nil { + fmt.Println(mc, stdout) + return 0.0, pkgerrors.WithStack(errors.New("could not find Go coverage report")) + } + coverage, err = strconv.ParseFloat(mc[1], 64) + if err != nil { + return 0.0, pkgerrors.WithStack(err) + } + + return coverage, nil } diff --git a/language/golang_test.go b/language/golang_test.go index c1d3fbf7..73857f09 100644 --- a/language/golang_test.go +++ b/language/golang_test.go @@ -55,6 +55,7 @@ func TestLanguageGolangExecute(t *testing.T) { RepositoryPath string RepositoryChange func(t *testing.T, repositoryPath string) + ExpectedCoverage float64 ExpectedError error ExpectedErrorText string } @@ -72,7 +73,7 @@ func TestLanguageGolangExecute(t *testing.T) { if tc.LanguageGolang == nil { tc.LanguageGolang = &LanguageGolang{} } - actualError := tc.LanguageGolang.Execute(repositoryPath) + actualCoverage, actualError := tc.LanguageGolang.Execute(repositoryPath) if tc.ExpectedError != nil { assert.ErrorIs(t, actualError, tc.ExpectedError) @@ -80,6 +81,7 @@ func TestLanguageGolangExecute(t *testing.T) { assert.ErrorContains(t, actualError, tc.ExpectedErrorText) } else { assert.NoError(t, actualError) + assert.Equal(t, tc.ExpectedCoverage, actualCoverage) } }) } @@ -110,6 +112,8 @@ func TestLanguageGolangExecute(t *testing.T) { } `)), 0660)) }, + + ExpectedCoverage: 100, }) validate(t, &testCase{ diff --git a/language/language.go b/language/language.go index 5bea798e..c92d580d 100644 --- a/language/language.go +++ b/language/language.go @@ -13,7 +13,7 @@ type Language interface { Files(repositoryPath string) (filePaths []string, err error) // Execute invokes the language specific testing on the given repository. - Execute(repositoryPath string) (err error) + Execute(repositoryPath string) (coverage float64, err error) } // Languages holds a register of all languages. diff --git a/model/symflower_test.go b/model/symflower_test.go index eca5bc40..fb98912c 100644 --- a/model/symflower_test.go +++ b/model/symflower_test.go @@ -22,6 +22,7 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) { RepositoryChange func(t *testing.T, repositoryPath string) FilePath string + ExpectedCoverage float64 ExpectedError error ExpectedErrorText string } @@ -47,7 +48,9 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) { assert.ErrorContains(t, actualErr, tc.ExpectedErrorText) } - require.NoError(t, tc.Language.Execute(repositoryPath)) + actualCoverage, err := tc.Language.Execute(repositoryPath) + require.NoError(t, err) + assert.Equal(t, tc.ExpectedCoverage, actualCoverage) }) } @@ -58,5 +61,7 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) { RepositoryPath: "../testdata/golang/plain/", FilePath: "plain.go", + + ExpectedCoverage: 100, }) } diff --git a/testdata/golang/plain/plain.go b/testdata/golang/plain/plain.go index d52fe06c..b644f9ea 100644 --- a/testdata/golang/plain/plain.go +++ b/testdata/golang/plain/plain.go @@ -1,4 +1,5 @@ package native func plain() { + var _ int = 0 }