feat(coverage): implement automated coverage validation tool (#14098)

.github/workflows/build-and-test.yml

@@ -225,6 +225,10 @@ jobs:
         uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # 5.5.1
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      - name: Validate coverage requirements
+        run: |
+          cd internal/cmd/checkcover
+          go run . -c ../../../coverage.txt -r ../../.. -v
 
   cross-build-collector:
     needs: [setup-environment]

.gitignore

@@ -2,6 +2,10 @@ bin/
 dist/
 /local
 
+# Compiled binaries
+cmd/checkcover/checkcover
+internal/cmd/checkcover/checkcover
+
 # GoLand IDEA
 /.idea/
 *.iml

cmd/mdatagen/internal/status.go

@@ -51,6 +51,7 @@ type Status struct {
 	Deprecation          DeprecationMap `mapstructure:"deprecation"`
 	CodeCovComponentID   string         `mapstructure:"codecov_component_id"`
 	DisableCodeCov       bool           `mapstructure:"disable_codecov_badge"`
+	CoverageMinimum      int            `mapstructure:"coverage_minimum"`
 }
 
 type DeprecationMap map[string]DeprecationInfo
@@ -134,6 +135,9 @@ func (s *Status) Validate() error {
 	if err := s.Deprecation.Validate(s.Stability); err != nil {
 		errs = errors.Join(errs, err)
 	}
+	if err := s.validateCoverageMinimum(); err != nil {
+		errs = errors.Join(errs, err)
+	}
 	return errs
 }
 
@@ -147,6 +151,13 @@ func (s *Status) validateClass() error {
 	return nil
 }
 
+func (s *Status) validateCoverageMinimum() error {
+	if s.CoverageMinimum < 0 || s.CoverageMinimum > 100 {
+		return fmt.Errorf("coverage_minimum must be between 0 and 100, got: %d", s.CoverageMinimum)
+	}
+	return nil
+}
+
 type StabilityMap map[component.StabilityLevel][]string
 
 func (ms StabilityMap) Validate() error {

cmd/mdatagen/internal/status_test.go

@@ -90,3 +90,72 @@ func TestSortedDistributions(t *testing.T) {
 		})
 	}
 }
+
+func TestStatus_ValidateCoverageMinimum(t *testing.T) {
+	tests := []struct {
+		name        string
+		coverage    int
+		expectError bool
+		errorMsg    string
+	}{
+		{
+			name:        "valid 0",
+			coverage:    0,
+			expectError: false,
+		},
+		{
+			name:        "valid 50",
+			coverage:    50,
+			expectError: false,
+		},
+		{
+			name:        "valid 80",
+			coverage:    80,
+			expectError: false,
+		},
+		{
+			name:        "valid 100",
+			coverage:    100,
+			expectError: false,
+		},
+		{
+			name:        "invalid -1",
+			coverage:    -1,
+			expectError: true,
+			errorMsg:    "coverage_minimum must be between 0 and 100",
+		},
+		{
+			name:        "invalid -10",
+			coverage:    -10,
+			expectError: true,
+			errorMsg:    "coverage_minimum must be between 0 and 100",
+		},
+		{
+			name:        "invalid 101",
+			coverage:    101,
+			expectError: true,
+			errorMsg:    "coverage_minimum must be between 0 and 100",
+		},
+		{
+			name:        "invalid 150",
+			coverage:    150,
+			expectError: true,
+			errorMsg:    "coverage_minimum must be between 0 and 100",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := &Status{
+				CoverageMinimum: tt.coverage,
+			}
+			err := s.validateCoverageMinimum()
+			if tt.expectError {
+				assert.Error(t, err)
+				assert.Contains(t, err.Error(), tt.errorMsg)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

cmd/mdatagen/metadata-schema.yaml

@@ -16,12 +16,24 @@ status:
   class: <receiver|processor|exporter|connector|extension|cmd|pkg|scraper|converter|provider>
   # Required: The stability of the component - See https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#stability-levels
   stability:
-    development: [<metrics|traces|logs|traces_to_metrics|metrics_to_metrics|logs_to_metrics|extension,converter,provider>]
-    alpha: [<metrics|traces|logs|traces_to_metrics|metrics_to_metrics|logs_to_metrics|extension,converter,provider>]
-    beta: [<metrics|traces|logs|traces_to_metrics|metrics_to_metrics|logs_to_metrics|extension,converter,provider>]
-    stable: [<metrics|traces|logs|traces_to_metrics|metrics_to_metrics|logs_to_metrics|extension,converter,provider>]
-    deprecated: [<metrics|traces|logs|traces_to_metrics|metrics_to_metrics|logs_to_metrics|extension,converter,provider>]
-    unmaintained: [<metrics|traces|logs|traces_to_metrics|metrics_to_metrics|logs_to_metrics|extension,converter,provider>]
+    development:
+      [
+        <metrics|traces|logs|traces_to_metrics|metrics_to_metrics|logs_to_metrics|extension,
+        converter,
+        provider>,
+      ]
+    alpha:
+      [
+        <metrics|traces|logs|traces_to_metrics|metrics_to_metrics|logs_to_metrics|extension,
+        converter,
+        provider>,
+      ]
+    beta:
+      [
+        <metrics|traces|logs|traces_to_metrics|metrics_to_metrics|logs_to_metrics|extension,
+        converter,
+        provider>,
+      ]
   # Required for deprecated components: The deprecation information for the deprecated components - See https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#deprecation-information
   deprecation:
     <component>:
@@ -36,29 +48,16 @@ status:
     active: [string]
     emeritus: [string]
   unsupported_platforms: [<linux|windows>]
+  # Optional: Minimum code coverage percentage required for this component.
+  # If not set, the default requirement based on stability level applies (e.g., 80% for stable).
+  # This can be used to set a higher requirement than the default. Setting it lower than the
+  # stability-based minimum has no effect.
+  coverage_minimum: int
 
 # Optional: OTel Semantic Conventions version that will be associated with the scraped metrics.
 # This attribute should be set for metrics compliant with OTel Semantic Conventions.
 sem_conv_version: 1.9.0
 
-# Optional: map of resource attribute definitions with the key being the attribute name.
-resource_attributes:
-  <attribute.name>:
-    # Required: whether the resource attribute is added the emitted metrics by default.
-    enabled: bool
-    # Required: description of the attribute.
-    description:
-    # Optional: array of attribute values if they are static values (currently, only string type is supported).
-    enum: [string]
-    # Required: attribute value type.
-    type: <string|int|double|bool|bytes|slice|map>
-    # Optional: warnings that will be shown to user under specified conditions.
-    warnings:
-      # A warning that will be displayed if the resource_attribute is enabled in user config.
-      # Should be used for deprecated default resource_attributes that will be removed soon.
-      if_enabled:
-      # A warning that will be displayed if `enabled` field is not set explicitly in user config.
-      # Should be used for resource_attributes that will be turned from default to optional or vice versa.
       if_enabled_not_set:
       # A warning that will be displayed if the resource_attribute is configured by user in any way.
       # Should be used for deprecated optional resource_attributes that will be removed soon.
@@ -133,9 +132,10 @@ metrics:
       monotonic: bool
       # Required for sum metric: whether reported values incorporate previous measurements
       # (cumulative) or not (delta).
-      aggregation_temporality: <delta|cumulative>
-       # Optional: Indicates the type the metric needs to be parsed from. If set, the generated
-       # functions will parse the value from string to value_type.
+      aggregation_temporality:
+        <delta|cumulative>
+        # Optional: Indicates the type the metric needs to be parsed from. If set, the generated
+        # functions will parse the value from string to value_type.
       input_type: string
     # Optional: array of attributes that were defined in the attributes section that are emitted by this metric.
     attributes: [string]
@@ -188,7 +188,6 @@ tests:
       top: [string] # Optional: array of strings representing functions that should be ignore via IgnoreTopFunction
       any: [string] # Optional: array of strings representing functions that should be ignore via IgnoreAnyFunction
 
-
 # Optional: map of metric names with the key being the metric name and value
 # being described below.
 telemetry:

docs/coverage-requirements.md

@@ -0,0 +1,175 @@
+# Code Coverage Requirements Guide for Component Authors
+
+## Overview
+
+As of this implementation, all OpenTelemetry Collector components are subject to automated code coverage validation based on their stability level.
+
+## Default Requirements
+
+### Stable Components
+
+- **Required**: 80% minimum code coverage
+- **Automatic**: Enforced by CI for all components with `stable` in their stability map
+
+### Other Stability Levels (Alpha, Beta, Development)
+
+- **Required**: No default minimum
+- **Optional**: Can set custom requirements (see below)
+
+## Setting Custom Coverage Requirements
+
+You can specify a higher coverage requirement for your component by adding the `coverage_minimum` field to your `metadata.yaml`:
+
+```yaml
+type: mycomponent
+status:
+  class: receiver
+  stability:
+    stable: [metrics, traces]
+  coverage_minimum: 85 # Require 85% coverage instead of the default 80%
+```
+
+### Important Notes
+
+1. **Higher Standards Only**: The `coverage_minimum` field can only increase the requirement, not decrease it. Setting it to a value lower than the stability-based minimum (80% for stable) has no effect.
+
+2. **Value Range**: Must be between 0 and 100. Values outside this range will cause validation errors.
+
+3. **Repository Minimum**: If a repository-wide minimum is set (via CI configuration), your component must meet whichever is higher: the stability-based minimum, the repository minimum, or your custom `coverage_minimum`.
+
+## Examples
+
+### Example 1: Stable Component with Default Coverage
+
+```yaml
+type: otlp
+status:
+  class: receiver
+  stability:
+    stable: [metrics, traces, logs]
+  # No coverage_minimum specified
+```
+
+**Result**: Must have ≥80% coverage
+
+### Example 2: Stable Component with Higher Requirement
+
+```yaml
+type: reliable
+status:
+  class: exporter
+  stability:
+    stable: [metrics]
+  coverage_minimum: 90
+```
+
+**Result**: Must have ≥90% coverage
+
+### Example 3: Alpha Component with Coverage Goal
+
+```yaml
+type: experimental
+status:
+  class: processor
+  stability:
+    alpha: [metrics]
+  coverage_minimum: 70
+```
+
+**Result**: Must have ≥70% coverage (helps ensure quality even at alpha stage)
+
+### Example 4: Beta Component Preparing for Stable
+
+```yaml
+type: maturing
+status:
+  class: connector
+  stability:
+    beta: [traces]
+  coverage_minimum: 80 # Preparing for stable promotion
+```
+
+**Result**: Must have ≥80% coverage (matching stable requirements)
+
+## Checking Your Component's Coverage
+
+### Locally
+
+You can check your component's coverage requirements and actual coverage:
+
+```bash
+# Generate coverage data
+make gotest-with-cover
+
+# Run the coverage validator
+cd internal/cmd/checkcover
+go run . -c ../../../coverage.txt -r ../../.. -v
+```
+
+### In CI
+
+Coverage validation runs automatically in CI after tests complete. If your component doesn't meet requirements, the build will fail with a clear error message:
+
+```
+❌ receiver/mycomponent: coverage 75.00% is below minimum 80% (stability: stable, module: go.opentelemetry.io/collector/receiver/mycomponent)
+```
+
+## Disabling Coverage Badge
+
+If you want to disable the Codecov badge in your component's README (not recommended), you can use:
+
+```yaml
+status:
+  disable_codecov_badge: true
+```
+
+**Note**: This only hides the badge; coverage validation still runs.
+
+## Best Practices
+
+1. **Set Realistic Goals**: If your component is stable, it should already meet or exceed 80% coverage. If not, work on adding tests before promoting to stable.
+
+2. **Incremental Improvement**: For alpha/beta components, consider setting a `coverage_minimum` that represents your goal, even if it's not required. This helps track progress.
+
+3. **Document Uncovered Code**: If certain code paths are difficult to test, document why in comments and consider if the design could be improved.
+
+4. **Review Coverage Reports**: Don't just aim for a percentage - ensure your tests are meaningful and cover important code paths.
+
+5. **CI First**: Always check that your changes pass coverage validation in CI before merging.
+
+## Troubleshooting
+
+### "Coverage X% is below minimum Y%"
+
+- **Solution**: Add more tests to your component to increase coverage
+- **Alternative**: If your component has difficult-to-test code, consider refactoring for testability
+
+### "Could not determine module path"
+
+- **Cause**: The tool couldn't find a `go.mod` file for your component
+- **Solution**: Ensure your component is in a proper Go module structure
+
+### "No coverage data found for module"
+
+- **Cause**: No test coverage was collected for your component
+- **Solution**: Ensure you have tests and they're being run by `make gotest-with-cover`
+
+## Getting Help
+
+If you encounter issues with coverage validation:
+
+1. Check the [Component Stability](../../docs/component-stability.md) document
+2. Review the [checkcover README](../../internal/cmd/checkcover/README.md)
+3. Ask in the #otel-collector Slack channel
+4. Open an issue on GitHub with the label `coverage`
+
+## Future Considerations
+
+The coverage requirements and validation tooling may evolve. Potential future changes include:
+
+- Default minimums for beta (e.g., 60%) and alpha (e.g., 40%) components
+- Coverage trend tracking
+- Exemptions for specific files or packages
+- HTML coverage reports
+
+Stay tuned to repository announcements for any changes to coverage requirements.

internal/cmd/checkcover/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common