feat: enhance completeness and validation reporting for datasets

Author: wiebe-vandendriessche

internal/completeness/completeness.go

@@ -14,6 +14,20 @@ type Report struct {
 
 	MissingRequired []metadata.Key
 	MissingOptional []metadata.Key
+
+	// Dataset-specific tracking
+	DatasetReports map[string]DatasetReport // key is dataset name/ref
+}
+
+type DatasetReport struct {
+	DatasetRef string // Reference to the dataset
+
+	Score  float64 // 0..1
+	Passed int
+	Total  int
+
+	MissingRequired []metadata.DatasetKey
+	MissingOptional []metadata.DatasetKey
 }
 
 func Check(bom *cdx.BOM) Report {
@@ -25,10 +39,27 @@ func Check(bom *cdx.BOM) Report {
 		missingOpt  []metadata.Key
 	)
 
+	// Check if datasets are referenced in model
+	datasetsReferenced := hasDatasetsReferenced(bom)
+
 	for _, spec := range metadata.Registry() {
 		if spec.Weight <= 0 {
 			continue
 		}
+
+		// Skip dataset field if no datasets are referenced
+		if spec.Key == metadata.ModelCardModelParametersDatasets && !datasetsReferenced {
+			// Only count as missing if no datasets are referenced
+			total++
+			max += spec.Weight
+			if spec.Required {
+				missingReq = append(missingReq, spec.Key)
+			} else {
+				missingOpt = append(missingOpt, spec.Key)
+			}
+			continue
+		}
+
 		total++
 		max += spec.Weight
 
@@ -55,7 +86,92 @@ func Check(bom *cdx.BOM) Report {
 		score = earned / max
 	}
 
-	return Report{
+	report := Report{
+		Score:           score,
+		Passed:          passed,
+		Total:           total,
+		MissingRequired: missingReq,
+		MissingOptional: missingOpt,
+		DatasetReports:  make(map[string]DatasetReport),
+	}
+
+	// Check dataset components if they exist
+	if bom.Components != nil && datasetsReferenced {
+		for _, comp := range *bom.Components {
+			if comp.Type == cdx.ComponentTypeData {
+				dsReport := CheckDataset(&comp)
+				report.DatasetReports[comp.Name] = dsReport
+			}
+		}
+	}
+
+	return report
+}
+
+// hasDatasetsReferenced checks if the model references any datasets
+func hasDatasetsReferenced(bom *cdx.BOM) bool {
+	if bom == nil || bom.Metadata == nil || bom.Metadata.Component == nil {
+		return false
+	}
+	comp := bom.Metadata.Component
+	if comp.ModelCard == nil || comp.ModelCard.ModelParameters == nil {
+		return false
+	}
+	mp := comp.ModelCard.ModelParameters
+	if mp.Datasets == nil || len(*mp.Datasets) == 0 {
+		return false
+	}
+	// Check if any dataset ref is non-empty
+	for _, ds := range *mp.Datasets {
+		if ds.Ref != "" {
+			return true
+		}
+	}
+	return false
+}
+
+// CheckDataset checks completeness of a single dataset component
+func CheckDataset(comp *cdx.Component) DatasetReport {
+	var (
+		earned, max float64
+		passed      int
+		total       int
+		missingReq  []metadata.DatasetKey
+		missingOpt  []metadata.DatasetKey
+	)
+
+	for _, spec := range metadata.DatasetRegistry() {
+		if spec.Weight <= 0 {
+			continue
+		}
+		total++
+		max += spec.Weight
+
+		ok := false
+		if spec.Present != nil {
+			ok = spec.Present(comp)
+		}
+
+		if ok {
+			passed++
+			earned += spec.Weight
+			continue
+		}
+
+		if spec.Required {
+			missingReq = append(missingReq, spec.Key)
+		} else {
+			missingOpt = append(missingOpt, spec.Key)
+		}
+	}
+
+	score := 0.0
+	if max > 0 {
+		score = earned / max
+	}
+
+	return DatasetReport{
+		DatasetRef:      comp.Name,
 		Score:           score,
 		Passed:          passed,
 		Total:           total,

internal/completeness/report.go

@@ -9,15 +9,31 @@ import (
 // PrintReport writes the report to the configured logger writer.
 // If no logger writer is configured, it produces no output.
 func PrintReport(r Report) {
-	logf("score=%.1f%% (%d/%d)", r.Score*100, r.Passed, r.Total)
+	logf("Model score=%.1f%% (%d/%d)", r.Score*100, r.Passed, r.Total)
 
 	if len(r.MissingRequired) > 0 {
 		logf("missing required: %s", joinKeys(r.MissingRequired))
 	}
 	if len(r.MissingOptional) > 0 {
 		logf("missing optional: %s", joinKeys(r.MissingOptional))
 	}
+
+	// Print dataset reports if any
+	if len(r.DatasetReports) > 0 {
+		logf("")
+		logf("Dataset Components:")
+		for dsName, dsReport := range r.DatasetReports {
+			logf("  %s: score=%.1f%% (%d/%d)", dsName, dsReport.Score*100, dsReport.Passed, dsReport.Total)
+			if len(dsReport.MissingRequired) > 0 {
+				logf("    missing required: %s", joinDatasetKeys(dsReport.MissingRequired))
+			}
+			if len(dsReport.MissingOptional) > 0 {
+				logf("    missing optional: %s", joinDatasetKeys(dsReport.MissingOptional))
+			}
+		}
+	}
 }
+
 func joinKeys(keys []metadata.Key) string {
 	if len(keys) == 0 {
 		return ""
@@ -31,3 +47,17 @@ func joinKeys(keys []metadata.Key) string {
 	}
 	return b.String()
 }
+
+func joinDatasetKeys(keys []metadata.DatasetKey) string {
+	if len(keys) == 0 {
+		return ""
+	}
+	var b strings.Builder
+	for i, k := range keys {
+		if i > 0 {
+			b.WriteString(", ")
+		}
+		b.WriteString(k.String())
+	}
+	return b.String()
+}