Remove scoring from within the evaluation cause the eval should not c…

…are how things are scored

cmd/eval-dev-quality/cmd/evaluate.go

@@ -21,7 +21,6 @@ import (
 	"golang.org/x/exp/maps"
 
 	"github.com/symflower/eval-dev-quality/evaluate"
-	"github.com/symflower/eval-dev-quality/evaluate/metrics"
 	"github.com/symflower/eval-dev-quality/evaluate/report"
 	evaltask "github.com/symflower/eval-dev-quality/evaluate/task"
 	"github.com/symflower/eval-dev-quality/language"
@@ -539,11 +538,9 @@ func (command *Evaluate) evaluateLocal(evaluationContext *evaluate.Context) (err
 	}
 
 	assessmentsPerModel := assessments.CollapseByModel()
-	_ = assessmentsPerModel.WalkByScore(func(model string, assessment metrics.Assessments, score uint64) (err error) {
-		command.logger.Printf("Evaluation score for %q: %s", model, assessment)
-
-		return nil
-	})
+	for _, modelID := range maps.Keys(assessmentsPerModel) {
+		command.logger.Printf("Evaluation score for %q: %s", modelID, assessmentsPerModel[modelID])
+	}
 
 	return nil
 }

evaluate/metrics/assessment.go

@@ -15,50 +15,45 @@ var (
 	allAssessmentKeys []AssessmentKey
 	// AllAssessmentKeysStrings returns all registered assessment keys as strings.
 	AllAssessmentKeysStrings []string
-
-	// multiplierPerAssessment holds the multipliers awarded for a specific assessment.
-	multiplierPerAssessment = map[AssessmentKey]uint64{}
 )
 
 // RegisterAssessmentKey registers a new assessment key.
-// If the multiplier for this assessment type is zero, it is ignored for the score computation.
-func RegisterAssessmentKey(key string, multiplier uint64) AssessmentKey {
+func RegisterAssessmentKey(key string) AssessmentKey {
 	assessment := AssessmentKey(key)
 	i := sort.SearchStrings(AllAssessmentKeysStrings, key)
 
 	allAssessmentKeys = slices.Insert(allAssessmentKeys, i, assessment)
 	AllAssessmentKeysStrings = slices.Insert(AllAssessmentKeysStrings, i, key)
-	multiplierPerAssessment[assessment] = multiplier
 
 	return assessment
 }
 
 var (
 	// AssessmentKeyFilesExecuted holds the successfully executed files.
-	AssessmentKeyFilesExecuted = RegisterAssessmentKey("files-executed", 1)
+	AssessmentKeyFilesExecuted = RegisterAssessmentKey("files-executed")
 	// AssessmentKeyFilesExecutedMaximumReachable holds the maximum theoretically reachable executed files.
-	AssessmentKeyFilesExecutedMaximumReachable = RegisterAssessmentKey("files-executed-maximum-reachable", 0)
+	AssessmentKeyFilesExecutedMaximumReachable = RegisterAssessmentKey("files-executed-maximum-reachable")
 	// AssessmentKeyProcessingTime holds the time in milliseconds that it took to complete the task.
-	AssessmentKeyProcessingTime = RegisterAssessmentKey("processing-time", 0)
+	AssessmentKeyProcessingTime = RegisterAssessmentKey("processing-time")
 
 	// AssessmentKeyCoverage counts execution coverage objects.
-	AssessmentKeyCoverage = RegisterAssessmentKey("coverage", 10)
+	AssessmentKeyCoverage = RegisterAssessmentKey("coverage")
 
 	// AssessmentKeyTestsPassing holds the percentage of passing tests.
-	AssessmentKeyTestsPassing = RegisterAssessmentKey("tests-passing", 10)
+	AssessmentKeyTestsPassing = RegisterAssessmentKey("tests-passing")
 
 	// AssessmentKeyResponseCharacterCount counts the number of characters of a response.
-	AssessmentKeyResponseCharacterCount = RegisterAssessmentKey("response-character-count", 0)
+	AssessmentKeyResponseCharacterCount = RegisterAssessmentKey("response-character-count")
 	// AssessmentKeyGenerateTestsForFileCharacterCount counts the number of characters of a generated test file.
-	AssessmentKeyGenerateTestsForFileCharacterCount = RegisterAssessmentKey("generate-tests-for-file-character-count", 0)
+	AssessmentKeyGenerateTestsForFileCharacterCount = RegisterAssessmentKey("generate-tests-for-file-character-count")
 
 	// AssessmentKeyResponseNoError indicates that a model responded without error.
-	AssessmentKeyResponseNoError = RegisterAssessmentKey("response-no-error", 1)
+	AssessmentKeyResponseNoError = RegisterAssessmentKey("response-no-error")
 	// AssessmentKeyResponseWithCode indicates that a model responded with code.
-	AssessmentKeyResponseWithCode = RegisterAssessmentKey("response-with-code", 1)
+	AssessmentKeyResponseWithCode = RegisterAssessmentKey("response-with-code")
 	// AssessmentKeyResponseNoExcess indicates that a model did not produce more content as requested.
 	// TODO Infer if a model produced "too much" code. https://github.com/symflower/eval-dev-quality/issues/44
-	AssessmentKeyResponseNoExcess = RegisterAssessmentKey("response-no-excess", 1)
+	AssessmentKeyResponseNoExcess = RegisterAssessmentKey("response-no-excess")
 )
 
 // Assessments holds a collection of numerical assessment metrics.
@@ -104,29 +99,14 @@ func Merge(a Assessments, b Assessments) (c Assessments) {
 	return c
 }
 
-// Score computes the score over all assessments in the collection.
-func (a Assessments) Score() (score uint64) {
-	if len(a) == 0 {
-		return 0
-	}
-
-	for key, value := range a {
-		if multiplierPerAssessment[key] != 0 {
-			score += value
-		}
-	}
-
-	return score
-}
-
-// Award yields the score points defined for the given key.
+// Award yields a score point.
 func (a Assessments) Award(key AssessmentKey) {
-	a[key] += multiplierPerAssessment[key]
+	a[key]++
 }
 
-// AwardPoints yields multiple score points defined for the given key.
-func (a Assessments) AwardPoints(key AssessmentKey, count uint64) {
-	a[key] += multiplierPerAssessment[key] * count
+// AwardMultiple yields multiple score points.
+func (a Assessments) AwardMultiple(key AssessmentKey, count uint64) {
+	a[key] += count
 }
 
 // String returns a string representation of the metrics.

evaluate/metrics/assessment_test.go

@@ -153,7 +153,7 @@ func TestAssessmentString(t *testing.T) {
 			AssessmentKeyResponseNoExcess:                   4,
 			AssessmentKeyResponseWithCode:                   5,
 			AssessmentKeyProcessingTime:                     200,
-			AssessmentKeyTestsPassing:                       70,
+			AssessmentKeyTestsPassing:                       7,
 		},
 
 		ExpectedString: "coverage=1, files-executed=2, files-executed-maximum-reachable=2, generate-tests-for-file-character-count=50, processing-time=200, response-character-count=100, response-no-error=3, response-no-excess=4, response-with-code=5, tests-passing=70",
@@ -237,44 +237,6 @@ func TestAssessmentsEqual(t *testing.T) {
 	})
 }
 
-func TestAssessmentsScore(t *testing.T) {
-	type testCase struct {
-		Name string
-
-		Assessments Assessments
-
-		ExpectedScore uint64
-	}
-
-	validate := func(t *testing.T, tc *testCase) {
-		t.Run(tc.Name, func(t *testing.T) {
-			actualScore := tc.Assessments.Score()
-
-			assert.Equal(t, tc.ExpectedScore, actualScore)
-		})
-	}
-
-	validate(t, &testCase{
-		Name: "Empty Assessment",
-
-		Assessments: NewAssessments(),
-
-		ExpectedScore: uint64(0),
-	})
-
-	validate(t, &testCase{
-		Name: "Values Assessment",
-
-		Assessments: Assessments{
-			AssessmentKeyFilesExecuted:  5,
-			AssessmentKeyCoverage:       4,
-			AssessmentKeyProcessingTime: 200,
-		},
-
-		ExpectedScore: uint64(9),
-	})
-}
-
 func TestCombineModelAndSymflowerFixAssessments(t *testing.T) {
 	type testCase struct {
 		Name string
@@ -309,21 +271,21 @@ func TestCombineModelAndSymflowerFixAssessments(t *testing.T) {
 		SymflowerFixAssessments: Assessments{
 			AssessmentKeyFilesExecuted:   1,
 			AssessmentKeyProcessingTime:  uint64(100),
-			AssessmentKeyCoverage:        10,
+			AssessmentKeyCoverage:        1,
 			AssessmentKeyResponseNoError: 1,
-			AssessmentKeyTestsPassing:    100,
+			AssessmentKeyTestsPassing:    10,
 		},
 
 		ExpectedAssessments: Assessments{
 			AssessmentKeyFilesExecuted:                      1,
 			AssessmentKeyProcessingTime:                     uint64(300),
-			AssessmentKeyCoverage:                           10,
+			AssessmentKeyCoverage:                           1,
 			AssessmentKeyResponseCharacterCount:             100,
 			AssessmentKeyGenerateTestsForFileCharacterCount: 50,
 			AssessmentKeyResponseNoError:                    0,
 			AssessmentKeyResponseWithCode:                   1,
 			AssessmentKeyResponseNoExcess:                   1,
-			AssessmentKeyTestsPassing:                       100,
+			AssessmentKeyTestsPassing:                       10,
 		},
 	})
 }

evaluate/report/collection.go

@@ -16,28 +16,6 @@ import (
 // AssessmentPerModel holds a collection of assessments per model id.
 type AssessmentPerModel map[string]metrics.Assessments
 
-// WalkByScore walks the given assessment metrics by their score.
-func (a AssessmentPerModel) WalkByScore(function func(model string, assessment metrics.Assessments, score uint64) error) (err error) {
-	models := maps.Keys(a)
-	sort.Strings(models)
-
-	scores := make(map[string]uint64, len(models))
-	for _, model := range models {
-		scores[model] = a[model].Score()
-	}
-	sort.SliceStable(models, func(i, j int) bool {
-		return scores[models[i]] < scores[models[j]]
-	})
-
-	for _, model := range models {
-		if err := function(model, a[model], scores[model]); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // AssessmentStore holds a collection of assessments per model per language and per repository.
 type AssessmentStore struct {
 	store map[model.Model]map[language.Language]map[string]map[task.Identifier]metrics.Assessments

evaluate/report/collection_test.go

@@ -4,7 +4,6 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
 	metricstesting "github.com/symflower/eval-dev-quality/evaluate/metrics/testing"
@@ -179,93 +178,6 @@ func TestAssessmentPerModelPerLanguagePerRepositoryWalk(t *testing.T) {
 	}
 }
 
-func TestWalkByScore(t *testing.T) {
-	type testCase struct {
-		Name string
-
-		AssessmentPerModel AssessmentPerModel
-
-		ExpectedModelOrder []string
-		ExpectedScoreOrder []uint64
-	}
-
-	validate := func(t *testing.T, tc *testCase) {
-		t.Run(tc.Name, func(t *testing.T) {
-			require.Equal(t, len(tc.ExpectedModelOrder), len(tc.ExpectedScoreOrder), "expected order needs equal lengths")
-
-			actualModelOrder := make([]string, 0, len(tc.ExpectedModelOrder))
-			actualAssessmentOrder := make([]metrics.Assessments, 0, len(tc.ExpectedModelOrder))
-			actualScoreOrder := make([]uint64, 0, len(tc.ExpectedScoreOrder))
-			assert.NoError(t, tc.AssessmentPerModel.WalkByScore(func(model string, assessment metrics.Assessments, score uint64) (err error) {
-				actualModelOrder = append(actualModelOrder, model)
-				actualAssessmentOrder = append(actualAssessmentOrder, assessment)
-				actualScoreOrder = append(actualScoreOrder, score)
-
-				return nil
-			}))
-
-			assert.Equal(t, tc.ExpectedModelOrder, actualModelOrder)
-			assert.Equal(t, tc.ExpectedScoreOrder, actualScoreOrder)
-			for i, model := range tc.ExpectedModelOrder {
-				assert.Equal(t, tc.AssessmentPerModel[model], actualAssessmentOrder[i])
-			}
-		})
-	}
-
-	validate(t, &testCase{
-		Name: "No Assessment",
-
-		AssessmentPerModel: AssessmentPerModel{},
-
-		ExpectedModelOrder: []string{},
-		ExpectedScoreOrder: []uint64{},
-	})
-
-	validate(t, &testCase{
-		Name: "Single Assessment",
-
-		AssessmentPerModel: AssessmentPerModel{
-			"modelA": metrics.Assessments{
-				metrics.AssessmentKeyFilesExecuted: 1,
-			},
-		},
-
-		ExpectedModelOrder: []string{
-			"modelA",
-		},
-		ExpectedScoreOrder: []uint64{
-			1,
-		},
-	})
-
-	validate(t, &testCase{
-		Name: "Multiple Assessments",
-
-		AssessmentPerModel: AssessmentPerModel{
-			"modelA": metrics.Assessments{
-				metrics.AssessmentKeyFilesExecuted: 1,
-			},
-			"modelB": metrics.Assessments{
-				metrics.AssessmentKeyFilesExecuted: 2,
-			},
-			"modelC": metrics.Assessments{
-				metrics.AssessmentKeyFilesExecuted: 3,
-			},
-		},
-
-		ExpectedModelOrder: []string{
-			"modelA",
-			"modelB",
-			"modelC",
-		},
-		ExpectedScoreOrder: []uint64{
-			1,
-			2,
-			3,
-		},
-	})
-}
-
 func TestAssessmentCollapseByModel(t *testing.T) {
 	type testCase struct {
 		Name string

evaluate/task/code-repair.go

@@ -104,7 +104,7 @@ func (t *CodeRepair) Run(ctx evaltask.Context) (repositoryAssessment map[evaltas
 		testsPassing := testResult.TestsPass
 		taskLogger.Printf("Executes tests with %d tests passing", testsPassing)
 		modelAssessment.Award(metrics.AssessmentKeyFilesExecuted)
-		modelAssessment.AwardPoints(metrics.AssessmentKeyTestsPassing, uint64(testsPassing))
+		modelAssessment.AwardMultiple(metrics.AssessmentKeyTestsPassing, uint64(testsPassing))
 	}
 
 	repositoryAssessment = map[evaltask.Identifier]metrics.Assessments{

evaluate/task/code-repair_test.go

@@ -76,7 +76,7 @@ func TestCodeRepairRun(t *testing.T) {
 						metrics.AssessmentKeyFilesExecuted:                 1,
 						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
 						metrics.AssessmentKeyResponseNoError:               1,
-						metrics.AssessmentKeyTestsPassing:                  40,
+						metrics.AssessmentKeyTestsPassing:                  4,
 					},
 				},
 				ValidateLog: func(t *testing.T, data string) {
@@ -140,7 +140,7 @@ func TestCodeRepairRun(t *testing.T) {
 						metrics.AssessmentKeyFilesExecuted:                 2,
 						metrics.AssessmentKeyFilesExecutedMaximumReachable: 2,
 						metrics.AssessmentKeyResponseNoError:               2,
-						metrics.AssessmentKeyTestsPassing:                  80,
+						metrics.AssessmentKeyTestsPassing:                  8,
 					},
 				},
 				ValidateLog: func(t *testing.T, data string) {
@@ -195,7 +195,7 @@ func TestCodeRepairRun(t *testing.T) {
 						metrics.AssessmentKeyFilesExecuted:                 1,
 						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
 						metrics.AssessmentKeyResponseNoError:               1,
-						metrics.AssessmentKeyTestsPassing:                  30,
+						metrics.AssessmentKeyTestsPassing:                  3,
 					},
 				},
 				ValidateLog: func(t *testing.T, data string) {
@@ -261,7 +261,7 @@ func TestCodeRepairRun(t *testing.T) {
 						metrics.AssessmentKeyFilesExecutedMaximumReachable: 2,
 						metrics.AssessmentKeyFilesExecuted:                 2,
 						metrics.AssessmentKeyResponseNoError:               2,
-						metrics.AssessmentKeyTestsPassing:                  60,
+						metrics.AssessmentKeyTestsPassing:                  6,
 					},
 				},
 				ValidateLog: func(t *testing.T, data string) {
@@ -310,7 +310,7 @@ func TestCodeRepairRun(t *testing.T) {
 						metrics.AssessmentKeyFilesExecuted:                 1,
 						metrics.AssessmentKeyFilesExecutedMaximumReachable: 1,
 						metrics.AssessmentKeyResponseNoError:               1,
-						metrics.AssessmentKeyTestsPassing:                  30,
+						metrics.AssessmentKeyTestsPassing:                  3,
 					},
 				},
 				ValidateLog: func(t *testing.T, data string) {
@@ -369,7 +369,7 @@ func TestCodeRepairRun(t *testing.T) {
 						metrics.AssessmentKeyFilesExecuted:                 2,
 						metrics.AssessmentKeyFilesExecutedMaximumReachable: 2,
 						metrics.AssessmentKeyResponseNoError:               2,
-						metrics.AssessmentKeyTestsPassing:                  60,
+						metrics.AssessmentKeyTestsPassing:                  6,
 					},
 				},
 				ValidateLog: func(t *testing.T, data string) {