From 73efec02f9c661b0c50f66b3d1877295e1a910ae Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Mon, 14 Oct 2024 15:36:59 +0200 Subject: [PATCH 1/8] Debug launch config to support relative paths and multiple arguments --- .vscode/launch.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 39a59d45..e2aeddcb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -21,9 +21,8 @@ "request": "launch", "mode": "auto", "program": "cmd/eval-dev-quality", - "args": [ - "${input:args}", - ] + "args": "${input:args}", + "cwd": "${workspaceFolder}" }, ], "inputs": [ @@ -54,7 +53,7 @@ "command": "memento.promptString", "args": { "id": "args", - "description": "Arguments? (Make sure to use absolute paths!)", + "description": "Arguments?", "default": "", }, }, From cd6495cfb0012e540890c9f3bdf41e56429f0495 Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Fri, 18 Oct 2024 09:20:50 +0200 Subject: [PATCH 2/8] Path-based detection of "plain" repositories so we can have other repositories with the "plain" suffix I.e. I (or a user) would want to add something like "spring-plain". We should not confuse it with an actual "plain" repository! Part of #365 --- evaluate/evaluate.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/evaluate/evaluate.go b/evaluate/evaluate.go index 2e96838b..41f244da 100644 --- a/evaluate/evaluate.go +++ b/evaluate/evaluate.go @@ -3,7 +3,6 @@ package evaluate import ( "os" "path/filepath" - "strings" "github.com/symflower/eval-dev-quality/evaluate/metrics" "github.com/symflower/eval-dev-quality/evaluate/report" @@ -207,7 +206,7 @@ func Evaluate(ctx *Context) (assessments *report.AssessmentStore) { } for _, repositoryPath := range relativeRepositoryPaths { // Do not include "plain" repositories in this step of the evaluation, because they have been checked with the common check before. - if !repositoriesLookup[repositoryPath] || strings.HasSuffix(repositoryPath, RepositoryPlainName) { + if !repositoriesLookup[repositoryPath] || filepath.Base(repositoryPath) == RepositoryPlainName { continue } From 4799dbfdc016a4518c8e3218d96cd92425788f1f Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Thu, 17 Oct 2024 13:42:10 +0200 Subject: [PATCH 3/8] Ignore paths in "write-tests" task repository configuration to enable multi-file tasks Part of #365 --- README.md | 11 +++++++ evaluate/task/repository.go | 17 +++++++++++ evaluate/task/repository_test.go | 52 ++++++++++++++++++++++++++++++++ evaluate/task/write-test.go | 19 ++++++++---- evaluate/task/write-test_test.go | 35 +++++++++++++++++++++ task/task.go | 2 ++ 6 files changed, 130 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7bd2b31c..8a88cbe6 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,17 @@ Each repository can contain a configuration file `repository.json` in its root d For the evaluation of the repository only the specified tasks are executed. If no `repository.json` file exists, all tasks are executed. +Depending on the task, it can be beneficial to exclude parts of the repository from explicit evaluation. To give a concrete example: Spring controller tests can never be executed on their own but need a supporting [`Application` class](https://docs.spring.io/spring-boot/reference/testing/spring-boot-applications.html#testing.spring-boot-applications.using-main). But such a file should never be used itself to prompt models for tests. Therefore, it can be excluded through the `repository.json` configuration: + +```json +{ + "tasks": ["write-tests"], + "ignore": ["src/main/java/com/example/Application.java"] +} +``` + +This `ignore` setting is currently only respected for the test generation task `write-tests`. + ## Tasks ### Task: Test Generation diff --git a/evaluate/task/repository.go b/evaluate/task/repository.go index 931a1ece..f8a10b46 100644 --- a/evaluate/task/repository.go +++ b/evaluate/task/repository.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" pkgerrors "github.com/pkg/errors" "github.com/zimmski/osutil" @@ -20,7 +21,10 @@ import ( // RepositoryConfiguration holds the configuration of a repository. type RepositoryConfiguration struct { + // Tasks holds the tasks supported by the repository. Tasks []task.Identifier + // IgnorePaths holds the relative paths that should be ignored when searching for cases. + IgnorePaths []string `json:"ignore,omitempty"` } // LoadRepositoryConfiguration loads a repository configuration from the given path. @@ -64,6 +68,19 @@ func (rc *RepositoryConfiguration) validate() (err error) { return nil } +// IsFilePathIgnored checks if the given relative file path is to be ignored when searching for cases. +func (rc *RepositoryConfiguration) IsFilePathIgnored(filePath string) bool { + filePath = filepath.Clean(filePath) + for _, ignoredFilePath := range rc.IgnorePaths { + ignoredFilePath = filepath.Clean(ignoredFilePath) + if strings.HasPrefix(filePath, ignoredFilePath) { + return true + } + } + + return false +} + // Repository holds data about a repository. type Repository struct { RepositoryConfiguration diff --git a/evaluate/task/repository_test.go b/evaluate/task/repository_test.go index f092c261..4fd26ff1 100644 --- a/evaluate/task/repository_test.go +++ b/evaluate/task/repository_test.go @@ -218,3 +218,55 @@ func TestRepositoryLoadConfiguration(t *testing.T) { }, }) } + +func TestRepositoryConfigurationIsFilePathIgnored(t *testing.T) { + type testCase struct { + Name string + + IgnoredPaths []string + FilePath string + + ExpectedBool bool + } + + validate := func(t *testing.T, tc *testCase) { + t.Run(tc.Name, func(t *testing.T) { + actualBool := (&RepositoryConfiguration{ + IgnorePaths: tc.IgnoredPaths, + }).IsFilePathIgnored(tc.FilePath) + + assert.Equal(t, tc.ExpectedBool, actualBool) + }) + } + + validate(t, &testCase{ + Name: "Exact Match", + + IgnoredPaths: []string{ + "foo/bar.txt", + }, + FilePath: "foo/bar.txt", + + ExpectedBool: true, + }) + validate(t, &testCase{ + Name: "No Match", + + IgnoredPaths: []string{ + "foo/bar.txt", + }, + FilePath: "foo/baz.txt", + + ExpectedBool: false, + }) + validate(t, &testCase{ + Name: "Folder", + + IgnoredPaths: []string{ + "foo", + }, + FilePath: "foo/bar.txt", + + ExpectedBool: true, + }) +} diff --git a/evaluate/task/write-test.go b/evaluate/task/write-test.go index d2cb2359..181303ff 100644 --- a/evaluate/task/write-test.go +++ b/evaluate/task/write-test.go @@ -57,13 +57,15 @@ func (t *WriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas withSymflowerTemplateAssessment := metrics.NewAssessments() withSymflowerTemplateAndFixAssessment := metrics.NewAssessments() - maximumReachableFiles := uint64(len(filePaths)) - modelAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles - withSymflowerFixAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles - withSymflowerTemplateAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles - withSymflowerTemplateAndFixAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles - + var maximumReachableFiles uint64 for _, filePath := range filePaths { + if ctx.Repository.IsFilePathIgnored(filePath) { + taskLogger.Printf("Ignoring file %q (as configured by the repository)", filePath) + + continue + } + maximumReachableFiles++ + // Handle this task case without a template. if err := ctx.Repository.Reset(ctx.Logger); err != nil { ctx.Logger.Panicf("ERROR: unable to reset temporary repository path: %s", err) @@ -114,6 +116,11 @@ func (t *WriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas withSymflowerTemplateAndFixAssessment.Add(templateWithSymflowerFixAssessmentFile) } + modelAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles + withSymflowerFixAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles + withSymflowerTemplateAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles + withSymflowerTemplateAndFixAssessment[metrics.AssessmentKeyFilesExecutedMaximumReachable] = maximumReachableFiles + repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{ IdentifierWriteTests: modelAssessment, IdentifierWriteTestsSymflowerFix: withSymflowerFixAssessment, diff --git a/evaluate/task/write-test_test.go b/evaluate/task/write-test_test.go index 45618ff2..0108e06a 100644 --- a/evaluate/task/write-test_test.go +++ b/evaluate/task/write-test_test.go @@ -298,6 +298,41 @@ func TestWriteTestsRun(t *testing.T) { }, }) } + + { + temporaryDirectoryPath := t.TempDir() + repositoryPath := filepath.Join(temporaryDirectoryPath, "golang", "plain") + require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "golang", "plain"), repositoryPath)) + require.NoError(t, os.WriteFile(filepath.Join(temporaryDirectoryPath, "golang", "plain", "repository.json"), []byte(bytesutil.StringTrimIndentations(` + { + "tasks": [ + "write-tests" + ], + "ignore": [ + "plain.go" + ] + } + `)), 0666)) + modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model") + validate(t, &tasktesting.TestCaseTask{ + Name: "Ignore Case", + + Model: modelMock, + Language: &golang.Language{}, + TestDataPath: temporaryDirectoryPath, + RepositoryPath: filepath.Join("golang", "plain"), + + ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ + IdentifierWriteTests: metrics.Assessments{}, + IdentifierWriteTestsSymflowerFix: metrics.Assessments{}, + IdentifierWriteTestsSymflowerTemplate: metrics.Assessments{}, + IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{}, + }, + ValidateLog: func(t *testing.T, data string) { + assert.Contains(t, data, "Ignoring file \"plain.go\" (as configured by the repository)") + }, + }) + } } func TestValidateWriteTestsRepository(t *testing.T) { diff --git a/task/task.go b/task/task.go index da3c04c5..ddd1e582 100644 --- a/task/task.go +++ b/task/task.go @@ -53,6 +53,8 @@ type Repository interface { // SupportedTasks returns the list of task identifiers the repository supports. SupportedTasks() (tasks []Identifier) + // IsFilePathIgnored checks if the given relative file path is to be ignored when searching for cases. + IsFilePathIgnored(filePath string) bool // Validate checks it the repository is well-formed. Validate(logger *log.Logger, language language.Language) (err error) From 6c32fab73dd4d2d4669a33dad78b2d4f6042aa04 Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Thu, 17 Oct 2024 14:43:30 +0200 Subject: [PATCH 4/8] Spring Boot plain "write-test" task case Part of #365 --- README.md | 2 +- evaluate/task/symflower.go | 1 + evaluate/task/write-test_test.go | 72 +++++++++++++++++++ testdata/java/spring-plain/pom.xml | 69 ++++++++++++++++++ testdata/java/spring-plain/repository.json | 4 ++ .../main/java/com/example/Application.java | 11 +++ .../example/controller/SomeController.java | 12 ++++ 7 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 testdata/java/spring-plain/pom.xml create mode 100644 testdata/java/spring-plain/repository.json create mode 100644 testdata/java/spring-plain/src/main/java/com/example/Application.java create mode 100644 testdata/java/spring-plain/src/main/java/com/example/controller/SomeController.java diff --git a/README.md b/README.md index 8a88cbe6..15310d61 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ Each repository can contain a configuration file `repository.json` in its root d For the evaluation of the repository only the specified tasks are executed. If no `repository.json` file exists, all tasks are executed. -Depending on the task, it can be beneficial to exclude parts of the repository from explicit evaluation. To give a concrete example: Spring controller tests can never be executed on their own but need a supporting [`Application` class](https://docs.spring.io/spring-boot/reference/testing/spring-boot-applications.html#testing.spring-boot-applications.using-main). But such a file should never be used itself to prompt models for tests. Therefore, it can be excluded through the `repository.json` configuration: +Depending on the task, it can be beneficial to exclude parts of the repository from explicit evaluation. To give a concrete example: Spring controller tests can never be executed on their own but need a supporting [`Application` class](https://docs.spring.io/spring-boot/reference/testing/spring-boot-applications.html#testing.spring-boot-applications.using-main). But [such a file](testdata/java/spring-plain/src/main/java/com/example/Application.java) should never be used itself to prompt models for tests. Therefore, it can be excluded through the `repository.json` configuration: ```json { diff --git a/evaluate/task/symflower.go b/evaluate/task/symflower.go index 1fd32d3c..188a6c9d 100644 --- a/evaluate/task/symflower.go +++ b/evaluate/task/symflower.go @@ -41,6 +41,7 @@ func symflowerTemplate(logger *log.Logger, repositoryPath string, language langu "--language", language.ID(), "--workspace", repositoryPath, "--test-style", "basic", + "--code-disable-fetch-dependencies", filePath, }, diff --git a/evaluate/task/write-test_test.go b/evaluate/task/write-test_test.go index 0108e06a..e2202043 100644 --- a/evaluate/task/write-test_test.go +++ b/evaluate/task/write-test_test.go @@ -333,6 +333,78 @@ func TestWriteTestsRun(t *testing.T) { }, }) } + + { + temporaryDirectoryPath := t.TempDir() + repositoryPath := filepath.Join(temporaryDirectoryPath, "java", "spring-plain") + require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "spring-plain"), repositoryPath)) + modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model") + modelMock.RegisterGenerateSuccess(t, filepath.Join("src", "test", "java", "com", "example", "controller", "SomeControllerTest.java"), bytesutil.StringTrimIndentations(` + package com.example.controller; + + import org.junit.jupiter.api.*; + import org.springframework.beans.factory.annotation.Autowired; + import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; + import org.springframework.test.web.servlet.MockMvc; + import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; + import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; + import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.view; + + @WebMvcTest(SomeController.class) + public class SomeControllerTest { + @Autowired + private MockMvc mockMvc; + + @Test + public void helloGet() throws Exception { + this.mockMvc.perform(get("/helloGet")) + .andExpect(status().isOk()) + .andExpect(view().name("get!")) + .andExpect(content().string("")); + } + } + `), metricstesting.AssessmentsWithProcessingTime) + + validate(t, &tasktesting.TestCaseTask{ + Name: "Spring Boot", + + Model: modelMock, + Language: &java.Language{}, + TestDataPath: temporaryDirectoryPath, + RepositoryPath: filepath.Join("java", "spring-plain"), + + ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ + IdentifierWriteTests: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerFix: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerTemplate: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, + }, + ValidateLog: func(t *testing.T, data string) { + assert.Equal(t, 2, strings.Count(data, "Starting SomeControllerTest using Java"), "Expected two successful Spring startup announcements (one bare and one for template)") + }, + }) + } } func TestValidateWriteTestsRepository(t *testing.T) { diff --git a/testdata/java/spring-plain/pom.xml b/testdata/java/spring-plain/pom.xml new file mode 100644 index 00000000..55790058 --- /dev/null +++ b/testdata/java/spring-plain/pom.xml @@ -0,0 +1,69 @@ + + + 4.0.0 + eval.dev.quality + test-java-plain + 1.0-SNAPSHOT + + 11 + 11 + UTF-8 + 2.7.9 + 2.7.9 + + + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot.version} + pom + import + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + + org.openclover + clover-maven-plugin + 4.5.2 + + false + false + false + true + true + true + true + true + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-test + test + + + diff --git a/testdata/java/spring-plain/repository.json b/testdata/java/spring-plain/repository.json new file mode 100644 index 00000000..004d4193 --- /dev/null +++ b/testdata/java/spring-plain/repository.json @@ -0,0 +1,4 @@ +{ + "tasks": ["write-tests"], + "ignore": ["src/main/java/com/example/Application.java"] +} diff --git a/testdata/java/spring-plain/src/main/java/com/example/Application.java b/testdata/java/spring-plain/src/main/java/com/example/Application.java new file mode 100644 index 00000000..fc8f3a65 --- /dev/null +++ b/testdata/java/spring-plain/src/main/java/com/example/Application.java @@ -0,0 +1,11 @@ +package com.example; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class Application { + public static void main(String[] args) { + SpringApplication.run(Application.class, args); + } +} diff --git a/testdata/java/spring-plain/src/main/java/com/example/controller/SomeController.java b/testdata/java/spring-plain/src/main/java/com/example/controller/SomeController.java new file mode 100644 index 00000000..033effb1 --- /dev/null +++ b/testdata/java/spring-plain/src/main/java/com/example/controller/SomeController.java @@ -0,0 +1,12 @@ +package com.example.controller; + +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.GetMapping; + +@Controller +public class SomeController { + @GetMapping("/helloGet") + public String helloGet() { + return "get!"; + } +} From 4fd1fced942f5c560469b4d9cf0f5aee33784e02 Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Fri, 18 Oct 2024 11:30:54 +0200 Subject: [PATCH 5/8] refactor, Move repository configuration to generic task definition to avoid having to update the interface every time Part of #365 --- cmd/eval-dev-quality/cmd/evaluate.go | 3 +- evaluate/evaluate.go | 2 +- evaluate/task/repository.go | 85 +++------------------------- evaluate/task/repository_test.go | 2 +- evaluate/task/task.go | 12 ++-- evaluate/task/write-test.go | 2 +- task/config.go | 81 ++++++++++++++++++++++++++ task/task.go | 6 +- util/iterable.go | 11 ++++ 9 files changed, 113 insertions(+), 91 deletions(-) create mode 100644 task/config.go create mode 100644 util/iterable.go diff --git a/cmd/eval-dev-quality/cmd/evaluate.go b/cmd/eval-dev-quality/cmd/evaluate.go index 8fdcfd93..674a3bd2 100644 --- a/cmd/eval-dev-quality/cmd/evaluate.go +++ b/cmd/eval-dev-quality/cmd/evaluate.go @@ -36,6 +36,7 @@ import ( openaiapi "github.com/symflower/eval-dev-quality/provider/openai-api" _ "github.com/symflower/eval-dev-quality/provider/openrouter" // Register provider. _ "github.com/symflower/eval-dev-quality/provider/symflower" // Register provider. + "github.com/symflower/eval-dev-quality/task" "github.com/symflower/eval-dev-quality/tools" "github.com/symflower/eval-dev-quality/util" ) @@ -317,7 +318,7 @@ func (command *Evaluate) Initialize(args []string) (evaluationContext *evaluate. command.logger.Panicf("ERROR: %s", err) } for _, r := range repositories { - config, err := evaltask.LoadRepositoryConfiguration(filepath.Join(command.TestdataPath, r)) + config, err := task.LoadRepositoryConfiguration(filepath.Join(command.TestdataPath, r), evaltask.AllIdentifiers) if err != nil { command.logger.Panicf("ERROR: %s", err) } diff --git a/evaluate/evaluate.go b/evaluate/evaluate.go index 41f244da..798a26e0 100644 --- a/evaluate/evaluate.go +++ b/evaluate/evaluate.go @@ -133,7 +133,7 @@ func Evaluate(ctx *Context) (assessments *report.AssessmentStore) { r.SetQueryAttempts(ctx.QueryAttempts) } - for _, taskIdentifier := range temporaryRepository.SupportedTasks() { + for _, taskIdentifier := range temporaryRepository.Configuration().Tasks { task, err := evaluatetask.ForIdentifier(taskIdentifier) if err != nil { logger.Fatal(err) diff --git a/evaluate/task/repository.go b/evaluate/task/repository.go index f8a10b46..9a07aed2 100644 --- a/evaluate/task/repository.go +++ b/evaluate/task/repository.go @@ -2,12 +2,10 @@ package task import ( "context" - "encoding/json" "errors" "fmt" "os" "path/filepath" - "strings" pkgerrors "github.com/pkg/errors" "github.com/zimmski/osutil" @@ -19,71 +17,9 @@ import ( "github.com/symflower/eval-dev-quality/util" ) -// RepositoryConfiguration holds the configuration of a repository. -type RepositoryConfiguration struct { - // Tasks holds the tasks supported by the repository. - Tasks []task.Identifier - // IgnorePaths holds the relative paths that should be ignored when searching for cases. - IgnorePaths []string `json:"ignore,omitempty"` -} - -// LoadRepositoryConfiguration loads a repository configuration from the given path. -func LoadRepositoryConfiguration(path string) (config *RepositoryConfiguration, err error) { - if osutil.FileExists(path) != nil { // If we don't get a valid file, assume it is a repository directory and target the default configuration file name. - path = filepath.Join(path, RepositoryConfigurationFileName) - } - - data, err := os.ReadFile(path) - if errors.Is(err, os.ErrNotExist) { - // Set default configuration. - return &RepositoryConfiguration{ - Tasks: AllIdentifiers, - }, nil - } else if err != nil { - return nil, pkgerrors.Wrap(err, path) - } - - config = &RepositoryConfiguration{} - if err := json.Unmarshal(data, &config); err != nil { - return nil, pkgerrors.Wrap(err, path) - } else if err := config.validate(); err != nil { - return nil, err - } - - return config, nil -} - -// validate validates the configuration. -func (rc *RepositoryConfiguration) validate() (err error) { - if len(rc.Tasks) == 0 { - return pkgerrors.Errorf("empty list of tasks in configuration") - } - - for _, taskIdentifier := range rc.Tasks { - if !LookupIdentifier[taskIdentifier] { - return pkgerrors.Errorf("task identifier %q unknown", taskIdentifier) - } - } - - return nil -} - -// IsFilePathIgnored checks if the given relative file path is to be ignored when searching for cases. -func (rc *RepositoryConfiguration) IsFilePathIgnored(filePath string) bool { - filePath = filepath.Clean(filePath) - for _, ignoredFilePath := range rc.IgnorePaths { - ignoredFilePath = filepath.Clean(ignoredFilePath) - if strings.HasPrefix(filePath, ignoredFilePath) { - return true - } - } - - return false -} - // Repository holds data about a repository. type Repository struct { - RepositoryConfiguration + task.RepositoryConfiguration // name holds the name of the repository. name string @@ -93,14 +29,11 @@ type Repository struct { var _ task.Repository = (*Repository)(nil) -// RepositoryConfigurationFileName holds the file name for a repository configuration. -const RepositoryConfigurationFileName = "repository.json" - // loadConfiguration loads the configuration from the dedicated configuration file. func (r *Repository) loadConfiguration() (err error) { - configurationFilePath := filepath.Join(r.dataPath, RepositoryConfigurationFileName) + configurationFilePath := filepath.Join(r.dataPath, task.RepositoryConfigurationFileName) - configuration, err := LoadRepositoryConfiguration(configurationFilePath) + configuration, err := task.LoadRepositoryConfiguration(configurationFilePath, AllIdentifiers) if err != nil { return err } @@ -120,14 +53,9 @@ func (r *Repository) DataPath() (dataPath string) { return r.dataPath } -// SupportedTasks returns the list of task identifiers the repository supports. -func (r *Repository) SupportedTasks() (tasks []task.Identifier) { - return r.Tasks -} - // Validate checks it the repository is well-formed. func (r *Repository) Validate(logger *log.Logger, language language.Language) (err error) { - for _, taskIdentifier := range r.SupportedTasks() { + for _, taskIdentifier := range r.RepositoryConfiguration.Tasks { switch taskIdentifier { case IdentifierCodeRepair: return validateCodeRepairRepository(logger, r.DataPath(), language) @@ -180,6 +108,11 @@ func (r *Repository) Reset(logger *log.Logger) (err error) { return nil } +// Configuration returns the configuration of a repository. +func (r *Repository) Configuration() *task.RepositoryConfiguration { + return &r.RepositoryConfiguration +} + // TemporaryRepository creates a temporary repository and initializes a git repo in it. func TemporaryRepository(logger *log.Logger, testDataPath string, repositoryPathRelative string) (repository *Repository, cleanup func(), err error) { repositoryPathAbsolute := filepath.Join(testDataPath, repositoryPathRelative) diff --git a/evaluate/task/repository_test.go b/evaluate/task/repository_test.go index 4fd26ff1..34315b0c 100644 --- a/evaluate/task/repository_test.go +++ b/evaluate/task/repository_test.go @@ -231,7 +231,7 @@ func TestRepositoryConfigurationIsFilePathIgnored(t *testing.T) { validate := func(t *testing.T, tc *testCase) { t.Run(tc.Name, func(t *testing.T) { - actualBool := (&RepositoryConfiguration{ + actualBool := (&task.RepositoryConfiguration{ IgnorePaths: tc.IgnoredPaths, }).IsFilePathIgnored(tc.FilePath) diff --git a/evaluate/task/task.go b/evaluate/task/task.go index d9325e08..0b28d5fa 100644 --- a/evaluate/task/task.go +++ b/evaluate/task/task.go @@ -10,24 +10,22 @@ import ( "github.com/symflower/eval-dev-quality/language" "github.com/symflower/eval-dev-quality/log" evaltask "github.com/symflower/eval-dev-quality/task" + "github.com/symflower/eval-dev-quality/util" ) var ( // AllIdentifiers holds all available task identifiers. AllIdentifiers []evaltask.Identifier - // LookupIdentifier holds a map of all available task identifiers. - LookupIdentifier = map[evaltask.Identifier]bool{} ) // registerIdentifier registers the given identifier and makes it available. func registerIdentifier(name string) (identifier evaltask.Identifier) { - identifier = evaltask.Identifier(name) - AllIdentifiers = append(AllIdentifiers, identifier) - - if _, ok := LookupIdentifier[identifier]; ok { + if _, ok := util.Set(AllIdentifiers)[identifier]; ok { panic(fmt.Sprintf("task identifier already registered: %s", identifier)) } - LookupIdentifier[identifier] = true + + identifier = evaltask.Identifier(name) + AllIdentifiers = append(AllIdentifiers, identifier) return identifier } diff --git a/evaluate/task/write-test.go b/evaluate/task/write-test.go index 181303ff..d41ac5b0 100644 --- a/evaluate/task/write-test.go +++ b/evaluate/task/write-test.go @@ -59,7 +59,7 @@ func (t *WriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas var maximumReachableFiles uint64 for _, filePath := range filePaths { - if ctx.Repository.IsFilePathIgnored(filePath) { + if ctx.Repository.Configuration().IsFilePathIgnored(filePath) { taskLogger.Printf("Ignoring file %q (as configured by the repository)", filePath) continue diff --git a/task/config.go b/task/config.go new file mode 100644 index 00000000..7bee5f17 --- /dev/null +++ b/task/config.go @@ -0,0 +1,81 @@ +package task + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + + pkgerrors "github.com/pkg/errors" + "github.com/symflower/eval-dev-quality/util" + "github.com/zimmski/osutil" +) + +// RepositoryConfiguration holds the configuration of a repository. +type RepositoryConfiguration struct { + // Tasks holds the tasks supported by the repository. + Tasks []Identifier + // IgnorePaths holds the relative paths that should be ignored when searching for cases. + IgnorePaths []string `json:"ignore,omitempty"` +} + +// RepositoryConfigurationFileName holds the file name for a repository configuration. +const RepositoryConfigurationFileName = "repository.json" + +// LoadRepositoryConfiguration loads a repository configuration from the given path. +func LoadRepositoryConfiguration(path string, defaultTasks []Identifier) (config *RepositoryConfiguration, err error) { + if osutil.FileExists(path) != nil { // If we don't get a valid file, assume it is a repository directory and target the default configuration file name. + path = filepath.Join(path, RepositoryConfigurationFileName) + } + + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) && len(defaultTasks) > 0 { + // Set default configuration. + return &RepositoryConfiguration{ + Tasks: defaultTasks, + }, nil + } else if err != nil { + return nil, pkgerrors.Wrap(err, path) + } + + config = &RepositoryConfiguration{} + if err := json.Unmarshal(data, &config); err != nil { + return nil, pkgerrors.Wrap(err, path) + } else if err := config.validate(defaultTasks); err != nil { + return nil, err + } + + return config, nil +} + +// validate validates the configuration. +func (rc *RepositoryConfiguration) validate(validTasks []Identifier) (err error) { + if len(rc.Tasks) == 0 { + return pkgerrors.Errorf("empty list of tasks in configuration") + } + + if len(validTasks) > 0 { + lookupTask := util.Set(validTasks) + for _, taskIdentifier := range rc.Tasks { + if !lookupTask[taskIdentifier] { + return pkgerrors.Errorf("task identifier %q unknown", taskIdentifier) + } + } + } + + return nil +} + +// IsFilePathIgnored checks if the given relative file path is to be ignored when searching for cases. +func (rc *RepositoryConfiguration) IsFilePathIgnored(filePath string) bool { + filePath = filepath.Clean(filePath) + for _, ignoredFilePath := range rc.IgnorePaths { + ignoredFilePath = filepath.Clean(ignoredFilePath) + if strings.HasPrefix(filePath, ignoredFilePath) { + return true + } + } + + return false +} diff --git a/task/task.go b/task/task.go index ddd1e582..4da03879 100644 --- a/task/task.go +++ b/task/task.go @@ -51,10 +51,8 @@ type Repository interface { // DataPath holds the absolute path to the repository. DataPath() (dataPath string) - // SupportedTasks returns the list of task identifiers the repository supports. - SupportedTasks() (tasks []Identifier) - // IsFilePathIgnored checks if the given relative file path is to be ignored when searching for cases. - IsFilePathIgnored(filePath string) bool + // Configuration returns the configuration of a repository. + Configuration() *RepositoryConfiguration // Validate checks it the repository is well-formed. Validate(logger *log.Logger, language language.Language) (err error) diff --git a/util/iterable.go b/util/iterable.go new file mode 100644 index 00000000..8c194509 --- /dev/null +++ b/util/iterable.go @@ -0,0 +1,11 @@ +package util + +// Set creates a set from a slice. +func Set[T comparable](s []T) map[T]bool { + set := make(map[T]bool, len(s)) + for _, i := range s { + set[i] = true + } + + return set +} From e0513c7de3ddcbd56ec841b192e4ceb9929483d4 Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Fri, 18 Oct 2024 11:54:02 +0200 Subject: [PATCH 6/8] refactor, Allow the "write-tests" test framework to be set within the task so it may be overwritten Part of #365 --- evaluate/task/write-test.go | 9 +++++-- model/llm/llm.go | 8 ++++--- model/llm/llm_test.go | 48 +++++++++++++++++++++++++++++-------- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/evaluate/task/write-test.go b/evaluate/task/write-test.go index d41ac5b0..d919ad2a 100644 --- a/evaluate/task/write-test.go +++ b/evaluate/task/write-test.go @@ -24,6 +24,8 @@ var _ evaltask.Task = (*WriteTests)(nil) type ArgumentsWriteTest struct { // Template holds the template data to base the tests onto. Template string + // TestFramework holds the test framework to use. + TestFramework string } // Identifier returns the write test task identifier. @@ -71,7 +73,9 @@ func (t *WriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas ctx.Logger.Panicf("ERROR: unable to reset temporary repository path: %s", err) } - modelAssessmentFile, withSymflowerFixAssessmentFile, ps, err := runModelAndSymflowerFix(ctx, taskLogger, modelCapability, dataPath, filePath, &ArgumentsWriteTest{}) + modelAssessmentFile, withSymflowerFixAssessmentFile, ps, err := runModelAndSymflowerFix(ctx, taskLogger, modelCapability, dataPath, filePath, &ArgumentsWriteTest{ + TestFramework: ctx.Language.TestFramework(), + }) problems = append(problems, ps...) if err != nil { return nil, problems, err @@ -105,7 +109,8 @@ func (t *WriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas } modelTemplateAssessmentFile, templateWithSymflowerFixAssessmentFile, ps, err := runModelAndSymflowerFix(ctx, taskLogger, modelCapability, dataPath, filePath, &ArgumentsWriteTest{ - Template: string(testTemplate), + Template: string(testTemplate), + TestFramework: ctx.Language.TestFramework(), }) problems = append(problems, ps...) if err != nil { diff --git a/model/llm/llm.go b/model/llm/llm.go index 15c02101..18a50542 100644 --- a/model/llm/llm.go +++ b/model/llm/llm.go @@ -83,11 +83,13 @@ type llmWriteTestSourceFilePromptContext struct { // Template holds the template data to base the tests onto. Template string + // TestFramework holds the test framework to use. + TestFramework string } // llmWriteTestForFilePromptTemplate is the template for generating an LLM test generation prompt. var llmWriteTestForFilePromptTemplate = template.Must(template.New("model-llm-write-test-for-file-prompt").Parse(bytesutil.StringTrimIndentations(` - Given the following {{ .Language.Name }} code file "{{ .FilePath }}" with package "{{ .ImportPath }}", provide a test file for this code{{ with $testFramework := .Language.TestFramework }} with {{ $testFramework }} as a test framework{{ end }}. + Given the following {{ .Language.Name }} code file "{{ .FilePath }}" with package "{{ .ImportPath }}", provide a test file for this code{{ with .TestFramework }} with {{ . }} as a test framework{{ end }}. The tests should produce 100 percent code coverage and must compile. The response must contain only the test code in a fenced code block and nothing else. @@ -211,7 +213,6 @@ func (m *Model) WriteTests(ctx model.Context) (assessment metrics.Assessments, e if !ok { return nil, pkgerrors.Errorf("unexpected type %T", ctx.Arguments) } - templateContent := arguments.Template data, err := os.ReadFile(filepath.Join(ctx.RepositoryPath, ctx.FilePath)) if err != nil { @@ -230,7 +231,8 @@ func (m *Model) WriteTests(ctx model.Context) (assessment metrics.Assessments, e ImportPath: importPath, }, - Template: templateContent, + Template: arguments.Template, + TestFramework: arguments.TestFramework, }).Format() if err != nil { return nil, err diff --git a/model/llm/llm_test.go b/model/llm/llm_test.go index b057b60c..b7254196 100644 --- a/model/llm/llm_test.go +++ b/model/llm/llm_test.go @@ -322,12 +322,12 @@ func TestFormatPromptContext(t *testing.T) { Language: &golang.Language{}, Code: bytesutil.StringTrimIndentations(` - package increment + package increment - func increment(i int) int - return i + 1 - } - `), + func increment(i int) int + return i + 1 + } + `), FilePath: filepath.Join("path", "to", "increment.go"), ImportPath: "increment", }, @@ -356,12 +356,12 @@ func TestFormatPromptContext(t *testing.T) { Language: &golang.Language{}, Code: bytesutil.StringTrimIndentations(` - package increment + package increment - func increment(i int) int - return i + 1 - } - `), + func increment(i int) int + return i + 1 + } + `), FilePath: filepath.Join("path", "to", "increment.go"), ImportPath: "increment", }, @@ -435,6 +435,34 @@ func TestFormatPromptContext(t *testing.T) { ` + "```" + ` `), }) + + validate(t, &testCase{ + Name: "Custom test framework", + + Context: &llmWriteTestSourceFilePromptContext{ + llmSourceFilePromptContext: llmSourceFilePromptContext{ + Language: &java.Language{}, + + Code: bytesutil.StringTrimIndentations(` + ${code} + `), + FilePath: filepath.Join("${path}"), + ImportPath: "${pkg}", + }, + + TestFramework: "JUnit 5 for Spring Boot", + }, + + ExpectedMessage: bytesutil.StringTrimIndentations(` + Given the following Java code file "${path}" with package "${pkg}", provide a test file for this code with JUnit 5 for Spring Boot as a test framework. + The tests should produce 100 percent code coverage and must compile. + The response must contain only the test code in a fenced code block and nothing else. + + ` + "```" + `java + ${code} + ` + "```" + ` + `), + }) }) validate(t, &testCase{ From 238e84c1d5547484c2da1cafdc019903913baddd Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Fri, 18 Oct 2024 14:05:25 +0200 Subject: [PATCH 7/8] Allow overwriting the "write-tests" task test framework from the repository configuration Part of #365 --- README.md | 13 +++++++++++++ evaluate/task/write-test.go | 9 +++++++-- evaluate/task/write-test_test.go | 7 ++++++- model/llm/llm_test.go | 4 ++-- model/testing/helper.go | 8 ++++++++ task/config.go | 6 ++++++ testdata/java/spring-plain/repository.json | 5 ++++- 7 files changed, 46 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 15310d61..3a179f7f 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,19 @@ Depending on the task, it can be beneficial to exclude parts of the repository f This `ignore` setting is currently only respected for the test generation task `write-tests`. +It is possible to configure some model prompt parameters through `repository.json`: + +```json +{ + "tasks": ["write-tests"], + "prompt": { + "test-framework": "JUnit 5 for Spring Boot" // Overwrite the default test framework in the prompt. + } +} +``` + +This `prompt.test-framework` setting is currently only respected for the test generation task `write-tests`. + ## Tasks ### Task: Test Generation diff --git a/evaluate/task/write-test.go b/evaluate/task/write-test.go index d919ad2a..8ce62e06 100644 --- a/evaluate/task/write-test.go +++ b/evaluate/task/write-test.go @@ -54,6 +54,11 @@ func (t *WriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas return nil, problems, pkgerrors.WithStack(err) } + testFramework := ctx.Language.TestFramework() + if ctx.Repository.Configuration().Prompt.TestFramework != "" { + testFramework = ctx.Repository.Configuration().Prompt.TestFramework + } + modelAssessment := metrics.NewAssessments() withSymflowerFixAssessment := metrics.NewAssessments() withSymflowerTemplateAssessment := metrics.NewAssessments() @@ -74,7 +79,7 @@ func (t *WriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas } modelAssessmentFile, withSymflowerFixAssessmentFile, ps, err := runModelAndSymflowerFix(ctx, taskLogger, modelCapability, dataPath, filePath, &ArgumentsWriteTest{ - TestFramework: ctx.Language.TestFramework(), + TestFramework: testFramework, }) problems = append(problems, ps...) if err != nil { @@ -110,7 +115,7 @@ func (t *WriteTests) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas modelTemplateAssessmentFile, templateWithSymflowerFixAssessmentFile, ps, err := runModelAndSymflowerFix(ctx, taskLogger, modelCapability, dataPath, filePath, &ArgumentsWriteTest{ Template: string(testTemplate), - TestFramework: ctx.Language.TestFramework(), + TestFramework: testFramework, }) problems = append(problems, ps...) if err != nil { diff --git a/evaluate/task/write-test_test.go b/evaluate/task/write-test_test.go index e2202043..1a7ffb34 100644 --- a/evaluate/task/write-test_test.go +++ b/evaluate/task/write-test_test.go @@ -17,6 +17,7 @@ import ( "github.com/symflower/eval-dev-quality/language/java" "github.com/symflower/eval-dev-quality/language/ruby" "github.com/symflower/eval-dev-quality/log" + "github.com/symflower/eval-dev-quality/model" modeltesting "github.com/symflower/eval-dev-quality/model/testing" evaltask "github.com/symflower/eval-dev-quality/task" "github.com/zimmski/osutil" @@ -339,7 +340,11 @@ func TestWriteTestsRun(t *testing.T) { repositoryPath := filepath.Join(temporaryDirectoryPath, "java", "spring-plain") require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "spring-plain"), repositoryPath)) modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model") - modelMock.RegisterGenerateSuccess(t, filepath.Join("src", "test", "java", "com", "example", "controller", "SomeControllerTest.java"), bytesutil.StringTrimIndentations(` + modelMock.RegisterGenerateSuccessValidateContext(t, func(t *testing.T, c model.Context) { + args, ok := c.Arguments.(*ArgumentsWriteTest) + require.Truef(t, ok, "unexpected type %T", c.Arguments) + assert.Equal(t, "JUnit 5 for Spring", args.TestFramework) + }, filepath.Join("src", "test", "java", "com", "example", "controller", "SomeControllerTest.java"), bytesutil.StringTrimIndentations(` package com.example.controller; import org.junit.jupiter.api.*; diff --git a/model/llm/llm_test.go b/model/llm/llm_test.go index b7254196..d3123bc2 100644 --- a/model/llm/llm_test.go +++ b/model/llm/llm_test.go @@ -450,11 +450,11 @@ func TestFormatPromptContext(t *testing.T) { ImportPath: "${pkg}", }, - TestFramework: "JUnit 5 for Spring Boot", + TestFramework: "JUnit 5 for Spring", }, ExpectedMessage: bytesutil.StringTrimIndentations(` - Given the following Java code file "${path}" with package "${pkg}", provide a test file for this code with JUnit 5 for Spring Boot as a test framework. + Given the following Java code file "${path}" with package "${pkg}", provide a test file for this code with JUnit 5 for Spring as a test framework. The tests should produce 100 percent code coverage and must compile. The response must contain only the test code in a fenced code block and nothing else. diff --git a/model/testing/helper.go b/model/testing/helper.go index 0d8d3731..8eb0f457 100644 --- a/model/testing/helper.go +++ b/model/testing/helper.go @@ -21,8 +21,16 @@ func NewMockModelNamed(t *testing.T, id string) *MockModel { // RegisterGenerateSuccess registers a mock call for successful generation. func (m *MockCapabilityWriteTests) RegisterGenerateSuccess(t *testing.T, filePath string, fileContent string, assessment metrics.Assessments) *mock.Call { + return m.RegisterGenerateSuccessValidateContext(t, nil, filePath, fileContent, assessment) +} + +// RegisterGenerateSuccessValidateContext registers a mock call for successful generation. +func (m *MockCapabilityWriteTests) RegisterGenerateSuccessValidateContext(t *testing.T, validateContext func(t *testing.T, c model.Context), filePath string, fileContent string, assessment metrics.Assessments) *mock.Call { return m.On("WriteTests", mock.Anything).Return(assessment, nil).Run(func(args mock.Arguments) { ctx, _ := args.Get(0).(model.Context) + if validateContext != nil { + validateContext(t, ctx) + } testFilePath := filepath.Join(ctx.RepositoryPath, filePath) require.NoError(t, os.MkdirAll(filepath.Dir(testFilePath), 0700)) require.NoError(t, os.WriteFile(testFilePath, []byte(fileContent), 0600)) diff --git a/task/config.go b/task/config.go index 7bee5f17..8129d9f7 100644 --- a/task/config.go +++ b/task/config.go @@ -18,6 +18,12 @@ type RepositoryConfiguration struct { Tasks []Identifier // IgnorePaths holds the relative paths that should be ignored when searching for cases. IgnorePaths []string `json:"ignore,omitempty"` + + // Prompt holds LLM prompt-related configuration. + Prompt struct { + // TestFramework overwrites the language-specific test framework to use. + TestFramework string `json:"test-framework,omitempty"` + } `json:",omitempty"` } // RepositoryConfigurationFileName holds the file name for a repository configuration. diff --git a/testdata/java/spring-plain/repository.json b/testdata/java/spring-plain/repository.json index 004d4193..1f816571 100644 --- a/testdata/java/spring-plain/repository.json +++ b/testdata/java/spring-plain/repository.json @@ -1,4 +1,7 @@ { "tasks": ["write-tests"], - "ignore": ["src/main/java/com/example/Application.java"] + "ignore": ["src/main/java/com/example/Application.java"], + "prompt": { + "test-framework": "JUnit 5 for Spring" + } } From e76cd17d40bd4cebe2ad486b858f5ed301f58e54 Mon Sep 17 00:00:00 2001 From: Simon Bauer Date: Fri, 18 Oct 2024 14:53:05 +0200 Subject: [PATCH 8/8] Allow custom validation for "write-test" task so one can ensure that tests for Spring Boot actually spin up Spring Boot Part of #365 --- README.md | 15 +++ evaluate/task/write-test.go | 4 +- evaluate/task/write-test_test.go | 147 +++++++++++++++------ language/golang/language.go | 2 + language/java/language.go | 2 + language/language.go | 2 + language/ruby/language.go | 2 + language/testing/language.go | 1 + task/config.go | 25 ++++ testdata/java/spring-plain/repository.json | 5 + 10 files changed, 160 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 3a179f7f..d6b7c85e 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,21 @@ It is possible to configure some model prompt parameters through `repository.jso This `prompt.test-framework` setting is currently only respected for the test generation task `write-tests`. +When task results are validated, some repositories might require custom logic. For example: generating tests for a Spring Boot project requires ensuring that the tests used an actual Spring context (i.e. Spring Boot was initialized when the tests were executed). Therefore, the `repository.json` supports adding rudimentary custom validation: + +```json +{ + "tasks": ["write-tests"], + "validation": { + "execution": { + "stdout": "Initializing Spring" // Ensure the string "Initializing Spring" is contained in the execution output. + } + } +} +``` + +This `validation.execution.stdout` setting is currently only respected for the test generation task `write-tests`. + ## Tasks ### Task: Test Generation diff --git a/evaluate/task/write-test.go b/evaluate/task/write-test.go index 8ce62e06..32890ead 100644 --- a/evaluate/task/write-test.go +++ b/evaluate/task/write-test.go @@ -168,7 +168,7 @@ func runModelAndSymflowerFix(ctx evaltask.Context, taskLogger *taskLogger, model problems = append(problems, ps...) if err != nil { problems = append(problems, pkgerrors.WithMessage(err, filePath)) - } else { + } else if ctx.Repository.Configuration().Validation.Execution.Validate(testResult.StdOut) { taskLogger.Printf("Executes tests with %d coverage objects", testResult.Coverage) modelAssessment.Award(metrics.AssessmentKeyFilesExecuted) modelAssessment.AwardPoints(metrics.AssessmentKeyCoverage, testResult.Coverage) @@ -179,7 +179,7 @@ func runModelAndSymflowerFix(ctx evaltask.Context, taskLogger *taskLogger, model problems = append(problems, ps...) if err != nil { problems = append(problems, err) - } else { + } else if ctx.Repository.Configuration().Validation.Execution.Validate(withSymflowerFixTestResult.StdOut) { ctx.Logger.Printf("with symflower repair: Executes tests with %d coverage objects", withSymflowerFixTestResult.Coverage) // Symflower was able to fix a failure so now update the assessment with the improved results. diff --git a/evaluate/task/write-test_test.go b/evaluate/task/write-test_test.go index 1a7ffb34..20112fc5 100644 --- a/evaluate/task/write-test_test.go +++ b/evaluate/task/write-test_test.go @@ -335,16 +335,17 @@ func TestWriteTestsRun(t *testing.T) { }) } - { - temporaryDirectoryPath := t.TempDir() - repositoryPath := filepath.Join(temporaryDirectoryPath, "java", "spring-plain") - require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "spring-plain"), repositoryPath)) - modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model") - modelMock.RegisterGenerateSuccessValidateContext(t, func(t *testing.T, c model.Context) { - args, ok := c.Arguments.(*ArgumentsWriteTest) - require.Truef(t, ok, "unexpected type %T", c.Arguments) - assert.Equal(t, "JUnit 5 for Spring", args.TestFramework) - }, filepath.Join("src", "test", "java", "com", "example", "controller", "SomeControllerTest.java"), bytesutil.StringTrimIndentations(` + t.Run("Spring Boot", func(t *testing.T) { + { + temporaryDirectoryPath := t.TempDir() + repositoryPath := filepath.Join(temporaryDirectoryPath, "java", "spring-plain") + require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "spring-plain"), repositoryPath)) + modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model") + modelMock.RegisterGenerateSuccessValidateContext(t, func(t *testing.T, c model.Context) { + args, ok := c.Arguments.(*ArgumentsWriteTest) + require.Truef(t, ok, "unexpected type %T", c.Arguments) + assert.Equal(t, "JUnit 5 for Spring", args.TestFramework) + }, filepath.Join("src", "test", "java", "com", "example", "controller", "SomeControllerTest.java"), bytesutil.StringTrimIndentations(` package com.example.controller; import org.junit.jupiter.api.*; @@ -371,45 +372,105 @@ func TestWriteTestsRun(t *testing.T) { } `), metricstesting.AssessmentsWithProcessingTime) - validate(t, &tasktesting.TestCaseTask{ - Name: "Spring Boot", + validate(t, &tasktesting.TestCaseTask{ + Name: "Spring Boot Test", - Model: modelMock, - Language: &java.Language{}, - TestDataPath: temporaryDirectoryPath, - RepositoryPath: filepath.Join("java", "spring-plain"), + Model: modelMock, + Language: &java.Language{}, + TestDataPath: temporaryDirectoryPath, + RepositoryPath: filepath.Join("java", "spring-plain"), - ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ - IdentifierWriteTests: metrics.Assessments{ - metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, - metrics.AssessmentKeyFilesExecuted: 1, - metrics.AssessmentKeyCoverage: 20, - metrics.AssessmentKeyResponseNoError: 1, + ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ + IdentifierWriteTests: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerFix: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerTemplate: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyFilesExecuted: 1, + metrics.AssessmentKeyCoverage: 20, + metrics.AssessmentKeyResponseNoError: 1, + }, }, - IdentifierWriteTestsSymflowerFix: metrics.Assessments{ - metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, - metrics.AssessmentKeyFilesExecuted: 1, - metrics.AssessmentKeyCoverage: 20, - metrics.AssessmentKeyResponseNoError: 1, + ValidateLog: func(t *testing.T, data string) { + assert.Equal(t, 2, strings.Count(data, "Starting SomeControllerTest using Java"), "Expected two successful Spring startup announcements (one bare and one for template)") }, - IdentifierWriteTestsSymflowerTemplate: metrics.Assessments{ - metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, - metrics.AssessmentKeyFilesExecuted: 1, - metrics.AssessmentKeyCoverage: 20, - metrics.AssessmentKeyResponseNoError: 1, + }) + } + { + temporaryDirectoryPath := t.TempDir() + repositoryPath := filepath.Join(temporaryDirectoryPath, "java", "spring-plain") + require.NoError(t, osutil.CopyTree(filepath.Join("..", "..", "testdata", "java", "spring-plain"), repositoryPath)) + modelMock := modeltesting.NewMockCapabilityWriteTestsNamed(t, "mocked-model") + modelMock.RegisterGenerateSuccessValidateContext(t, func(t *testing.T, c model.Context) { + args, ok := c.Arguments.(*ArgumentsWriteTest) + require.Truef(t, ok, "unexpected type %T", c.Arguments) + assert.Equal(t, "JUnit 5 for Spring", args.TestFramework) + }, filepath.Join("src", "test", "java", "com", "example", "controller", "SomeControllerTest.java"), bytesutil.StringTrimIndentations(` + package com.example.controller; + + import com.example.controller.SomeController; + import org.junit.jupiter.api.Test; + + import static org.junit.jupiter.api.Assertions.assertEquals; + + class SomeControllerTests { + + @Test // Normal JUnit tests instead of Spring Boot. + void helloGet() { + SomeController controller = new SomeController(); + String result = controller.helloGet(); + assertEquals("get!", result); + } + } + `), metricstesting.AssessmentsWithProcessingTime) + + validate(t, &tasktesting.TestCaseTask{ + Name: "Plain JUnit Test", + + Model: modelMock, + Language: &java.Language{}, + TestDataPath: temporaryDirectoryPath, + RepositoryPath: filepath.Join("java", "spring-plain"), + + ExpectedRepositoryAssessment: map[evaltask.Identifier]metrics.Assessments{ + IdentifierWriteTests: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerFix: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerTemplate: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyResponseNoError: 1, + }, + IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{ + metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, + metrics.AssessmentKeyResponseNoError: 1, + }, }, - IdentifierWriteTestsSymflowerTemplateSymflowerFix: metrics.Assessments{ - metrics.AssessmentKeyFilesExecutedMaximumReachable: 1, - metrics.AssessmentKeyFilesExecuted: 1, - metrics.AssessmentKeyCoverage: 20, - metrics.AssessmentKeyResponseNoError: 1, + ValidateLog: func(t *testing.T, data string) { + assert.Contains(t, data, "Tests run: 1") // Tests are running but they are not Spring Boot. }, - }, - ValidateLog: func(t *testing.T, data string) { - assert.Equal(t, 2, strings.Count(data, "Starting SomeControllerTest using Java"), "Expected two successful Spring startup announcements (one bare and one for template)") - }, - }) - } + }) + } + }) } func TestValidateWriteTestsRepository(t *testing.T) { diff --git a/language/golang/language.go b/language/golang/language.go index 88862d85..4a38aff4 100644 --- a/language/golang/language.go +++ b/language/golang/language.go @@ -109,6 +109,8 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test testResult = &language.TestResult{ TestsTotal: uint(testsTotal), TestsPass: uint(testsPass), + + StdOut: commandOutput, } testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) if err != nil { diff --git a/language/java/language.go b/language/java/language.go index 6a3b76f4..447340e2 100644 --- a/language/java/language.go +++ b/language/java/language.go @@ -107,6 +107,8 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test testResult = &language.TestResult{ TestsTotal: uint(testsTotal), TestsPass: uint(testsPass), + + StdOut: commandOutput, } testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) diff --git a/language/language.go b/language/language.go index 550657a8..2c939cbf 100644 --- a/language/language.go +++ b/language/language.go @@ -125,6 +125,8 @@ type TestResult struct { TestsPass uint Coverage uint64 + + StdOut string } // PassingTestsPercentage returns the percentage of passing tests. diff --git a/language/ruby/language.go b/language/ruby/language.go index 78c49ec2..cd6c7df2 100644 --- a/language/ruby/language.go +++ b/language/ruby/language.go @@ -101,6 +101,8 @@ func (l *Language) ExecuteTests(logger *log.Logger, repositoryPath string) (test testResult = &language.TestResult{ TestsTotal: uint(testsTotal), TestsPass: uint(testsPass), + + StdOut: commandOutput, } testResult.Coverage, err = language.CoverageObjectCountOfFile(logger, coverageFilePath) diff --git a/language/testing/language.go b/language/testing/language.go index 07196ed1..d3b993a9 100644 --- a/language/testing/language.go +++ b/language/testing/language.go @@ -57,6 +57,7 @@ func (tc *TestCaseExecuteTests) Validate(t *testing.T) { assert.ErrorContains(t, actualError, tc.ExpectedErrorText) } else { assert.NoError(t, actualError) + actualTestResult.StdOut = "" assert.Equal(t, tc.ExpectedTestResult, actualTestResult) } }) diff --git a/task/config.go b/task/config.go index 8129d9f7..a1c994d7 100644 --- a/task/config.go +++ b/task/config.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "regexp" "strings" pkgerrors "github.com/pkg/errors" @@ -24,6 +25,17 @@ type RepositoryConfiguration struct { // TestFramework overwrites the language-specific test framework to use. TestFramework string `json:"test-framework,omitempty"` } `json:",omitempty"` + + // Validation holds quality gates for evaluation. + Validation struct { + Execution RepositoryConfigurationExecution `json:",omitempty"` + } +} + +// RepositoryConfigurationExecution execution-related quality gates for evaluation. +type RepositoryConfigurationExecution struct { + // StdOutRE holds a regular expression that must be part of execution standard output. + StdOutRE string `json:"stdout,omitempty"` } // RepositoryConfigurationFileName holds the file name for a repository configuration. @@ -70,6 +82,10 @@ func (rc *RepositoryConfiguration) validate(validTasks []Identifier) (err error) } } + if _, err := regexp.Compile(rc.Validation.Execution.StdOutRE); err != nil { + return pkgerrors.WithMessagef(err, "invalid regular expression %q", rc.Validation.Execution.StdOutRE) + } + return nil } @@ -85,3 +101,12 @@ func (rc *RepositoryConfiguration) IsFilePathIgnored(filePath string) bool { return false } + +// Validate validates execution outcomes against the configured quality gates. +func (e *RepositoryConfigurationExecution) Validate(stdout string) bool { + if e.StdOutRE != "" { + return regexp.MustCompile(e.StdOutRE).MatchString(stdout) + } + + return true +} diff --git a/testdata/java/spring-plain/repository.json b/testdata/java/spring-plain/repository.json index 1f816571..7a17122e 100644 --- a/testdata/java/spring-plain/repository.json +++ b/testdata/java/spring-plain/repository.json @@ -3,5 +3,10 @@ "ignore": ["src/main/java/com/example/Application.java"], "prompt": { "test-framework": "JUnit 5 for Spring" + }, + "validation": { + "execution": { + "stdout": "Initializing Spring" + } } }