test: improve coverage

.github/workflows/test.yml

@@ -26,12 +26,27 @@ jobs:
           name: coverage
           path: coverage.out
 
+  pretest:
+    runs-on: ubuntu-latest
+    outputs:
+      plugins: ${{ steps.set-matrix.outputs.plugins }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - id: set-matrix
+        run: |
+          echo "plugins=$(find plugins -mindepth 2 -maxdepth 2 -type d | sed 's/plugins\///' | awk 'BEGIN{printf "["} {printf "%s\"%s\"",sep,$0; sep=","} END{print ",\".\"]"}')" >> $GITHUB_OUTPUT
+
   plugins-test:
+    needs: pretest
     runs-on: ubuntu-latest
     if: |
       github.ref == 'refs/heads/main' || 
       github.event_name == 'pull_request' ||
       github.event_name == 'workflow_dispatch'
+    strategy:
+      matrix:
+        plugins: ${{ fromJson(needs.pretest.outputs.plugins) }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -43,12 +58,12 @@ jobs:
       - name: Install dependencies
         run: sudo apt-get install build-essential
       - name: Run Test
-        run: make test-plugins
+        run: make test-plugins PLUGIN=${{ matrix.plugins }}
       - name: Upload coverage artifact
         uses: actions/upload-artifact@v3
         with:
           name: coverage-plugins
-          path: coverage-plugins.out
+          path: coverage-plugins*.out
 
   coverage:
     runs-on: ubuntu-latest
@@ -74,4 +89,4 @@ jobs:
           COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           go install github.com/mattn/[email protected]
-          goveralls -coverprofile=coverage.out,coverage-plugins.out -service=github
+          goveralls -coverprofile=$(ls -1 coverage*.out | paste -sd "," -) -service=github

Makefile

@@ -19,14 +19,14 @@ copy-config:
 	cp ./config/meteor.yaml.sample ./meteor.yaml
 
 test:
-	go test ./... -coverprofile=coverage.out
+	go test $(shell go list ./... | grep -v 'test\|mocks\|plugins\|v1beta2\|cmd') -coverprofile=coverage.out
 
 test-e2e:
 	go test ./test/e2e -tags=integration -count=1
 
 test-plugins:
 	@echo " > Testing plugins with tag 'plugins'"
-	go test ./plugins... -tags=plugins -coverprofile=coverage-plugins.out -parallel=1
+	go test $(if $(filter .,$(PLUGIN)),./plugins,$(if $(PLUGIN),./plugins/$(PLUGIN)/...,./plugins/...)) -tags=plugins -coverprofile=coverage-plugins$(subst .,root,$(subst /,-,$(if $(PLUGIN),-$(PLUGIN),))).out -parallel=1
 
 test-coverage: # test test-plugins
 	cp coverage.out coverage-all.out

agent/stream.go

@@ -4,16 +4,17 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/pkg/errors"
 	"github.com/raystack/meteor/models"
 )
 
-type streamMiddleware func(src models.Record) (dst models.Record, err error)
-type subscriber struct {
-	callback  func([]models.Record) error
-	channel   chan models.Record
-	batchSize int
-}
+type (
+	streamMiddleware func(src models.Record) (dst models.Record, err error)
+	subscriber       struct {
+		callback  func([]models.Record) error
+		channel   chan models.Record
+		batchSize int
+	}
+)
 
 type stream struct {
 	middlewares []streamMiddleware
@@ -94,7 +95,7 @@ func (s *stream) broadcast() error {
 func (s *stream) push(data models.Record) {
 	data, err := s.runMiddlewares(data)
 	if err != nil {
-		s.closeWithError(errors.Wrap(err, "emitter: error running middleware"))
+		s.closeWithError(fmt.Errorf("emitter: error running middleware: %w", err))
 		return
 	}
 

config/config_test.go

@@ -0,0 +1,71 @@
+package config_test
+
+import (
+	"testing"
+
+	"github.com/raystack/meteor/config"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoad(t *testing.T) {
+	type args struct {
+		configFile string
+	}
+	tests := []struct {
+		name        string
+		args        args
+		expected    config.Config
+		expectedErr string
+	}{
+		{
+			name: "should return a config",
+			args: args{
+				configFile: "testdata/valid-config.yaml",
+			},
+			expected: config.Config{
+				LogLevel:                    "info",
+				StatsdEnabled:               false,
+				StatsdHost:                  "localhost:8125",
+				StatsdPrefix:                "meteor",
+				MaxRetries:                  5,
+				RetryInitialIntervalSeconds: 5,
+				StopOnSinkError:             false,
+			},
+		},
+		{
+			name: "config file not found",
+			args: args{
+				configFile: "not-found.yaml",
+			},
+			expected: config.Config{
+				LogLevel:                    "info",
+				StatsdEnabled:               false,
+				StatsdHost:                  "localhost:8125",
+				StatsdPrefix:                "meteor",
+				MaxRetries:                  5,
+				RetryInitialIntervalSeconds: 5,
+			},
+			expectedErr: "",
+		},
+		{
+			name: "config invalid",
+			args: args{
+				configFile: "testdata/invalid-config.yaml",
+			},
+			expected:    config.Config{},
+			expectedErr: "unable to load config to struct",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual, err := config.Load(tt.args.configFile)
+			if tt.expectedErr != "" {
+				assert.ErrorContains(t, err, tt.expectedErr)
+				return
+			}
+
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expected, actual)
+		})
+	}
+}

config/testdata/invalid-config.yaml

@@ -0,0 +1 @@
+STATSD_ENABLED: not-a-boolean

config/testdata/valid-config.yaml

@@ -0,0 +1,7 @@
+LOG_LEVEL: info
+STATSD_ENABLED: false
+STATSD_HOST: "localhost:8125"
+STATSD_PREFIX: meteor
+MAX_RETRIES: 5
+RETRY_INITIAL_INTERVAL_SECONDS: 5
+STOP_ON_SINK_ERROR: false

generator/recipe_test.go

@@ -0,0 +1,202 @@
+package generator_test
+
+import (
+	"bytes"
+	_ "embed"
+	"testing"
+
+	"github.com/raystack/meteor/generator"
+	"github.com/raystack/meteor/plugins"
+	"github.com/raystack/meteor/registry"
+	"github.com/raystack/meteor/test/mocks"
+	"github.com/stretchr/testify/assert"
+)
+
+var recipeVersions = [1]string{"v1beta1"}
+
+func TestRecipe(t *testing.T) {
+	var err error
+	err = registry.Extractors.Register("test-extractor", func() plugins.Extractor {
+		extr := mocks.NewExtractor()
+		mockInfo := plugins.Info{
+			Description: "Mock Extractor 1",
+		}
+		extr.On("Info").Return(mockInfo, nil).Once()
+		return extr
+	})
+	assert.NoError(t, err)
+
+	err = registry.Sinks.Register("test-sink", func() plugins.Syncer {
+		mockSink := mocks.NewSink()
+		mockInfo := plugins.Info{
+			Description: "Mock Sink 1",
+		}
+		mockSink.On("Info").Return(mockInfo, nil).Once()
+		return mockSink
+	})
+	assert.NoError(t, err)
+
+	err = registry.Processors.Register("test-processor", func() plugins.Processor {
+		mockProcessor := mocks.NewProcessor()
+		mockInfo := plugins.Info{
+			Description: "Mock Processor 1",
+		}
+		mockProcessor.On("Info").Return(mockInfo, nil).Once()
+		return mockProcessor
+	})
+	assert.NoError(t, err)
+
+	type args struct {
+		p generator.RecipeParams
+	}
+	tests := []struct {
+		name        string
+		args        args
+		expected    *generator.TemplateData
+		expectedErr string
+	}{
+		{
+			name: "success with minimal params",
+			args: args{
+				p: generator.RecipeParams{
+					Name: "test-name",
+				},
+			},
+			expected: &generator.TemplateData{
+				Name:    "test-name",
+				Version: recipeVersions[len(recipeVersions)-1],
+			},
+		},
+		{
+			name: "success with full params",
+			args: args{
+				p: generator.RecipeParams{
+					Name:       "test-name",
+					Source:     "test-extractor",
+					Sinks:      []string{"test-sink"},
+					Processors: []string{"test-processor"},
+				},
+			},
+			expected: &generator.TemplateData{
+				Name:    "test-name",
+				Version: recipeVersions[len(recipeVersions)-1],
+				Source: struct {
+					Name         string
+					Scope        string
+					SampleConfig string
+				}{
+					Name: "test-extractor",
+				},
+				Sinks: map[string]string{
+					"test-sink": "",
+				},
+				Processors: map[string]string{
+					"test-processor": "",
+				},
+			},
+		},
+		{
+			name: "error with invalid source",
+			args: args{
+				p: generator.RecipeParams{
+					Name:   "test-name",
+					Source: "invalid-source",
+				},
+			},
+			expected:    nil,
+			expectedErr: "provide extractor information: could not find extractor",
+		},
+		{
+			name: "error with invalid sinks",
+			args: args{
+				p: generator.RecipeParams{
+					Name:  "test-name",
+					Sinks: []string{"invalid-sink"},
+				},
+			},
+			expected:    nil,
+			expectedErr: "provide sink information: could not find sink",
+		},
+		{
+			name: "error with invalid processors",
+			args: args{
+				p: generator.RecipeParams{
+					Name:       "test-name",
+					Processors: []string{"invalid-processor"},
+				},
+			},
+			expected:    nil,
+			expectedErr: "provide processor information: could not find processor",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual, err := generator.Recipe(tt.args.p)
+			if tt.expectedErr != "" {
+				assert.ErrorContains(t, err, tt.expectedErr)
+				return
+			}
+			assert.NoError(t, err)
+
+			assert.Equal(t, actual, tt.expected)
+		})
+	}
+}
+
+func TestRecipeWriteTo(t *testing.T) {
+	type args struct {
+		p generator.RecipeParams
+	}
+	tests := []struct {
+		name           string
+		args           args
+		expectedWriter string
+		expectedErr    string
+	}{
+		{
+			name: "success with minimal params",
+			args: args{
+				p: generator.RecipeParams{
+					Name: "test-name",
+				},
+			},
+			expectedWriter: `name: test-name
+version: v1beta1
+source:
+  name: 
+  config:     
+    
+`,
+			expectedErr: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			writer := &bytes.Buffer{}
+			err := generator.RecipeWriteTo(tt.args.p, writer)
+			if tt.expectedErr != "" {
+				assert.ErrorContains(t, err, tt.expectedErr)
+				return
+			}
+			assert.Equal(t, tt.expectedWriter, writer.String())
+		})
+	}
+}
+
+func TestGetRecipeVersions(t *testing.T) {
+	tests := []struct {
+		name     string
+		expected [1]string
+	}{
+		{
+			name:     "success",
+			expected: recipeVersions,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := generator.GetRecipeVersions()
+			assert.Equal(t, tt.expected, actual)
+		})
+	}
+}