From 85d659a733ba0ca491ba336c2ad7e85b03ef80f0 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 9 Oct 2025 14:23:21 -0400 Subject: [PATCH 01/36] initial structure of command --- cmd/filter.go | 99 ++++++++++++++++++++++++++++++++++++++ cmd/root.go | 1 + internal/cobraext/flags.go | 17 ++++++- 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 cmd/filter.go diff --git a/cmd/filter.go b/cmd/filter.go new file mode 100644 index 0000000000..7bc9ccc1ba --- /dev/null +++ b/cmd/filter.go @@ -0,0 +1,99 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "fmt" + + "github.com/Masterminds/semver/v3" + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/spf13/cobra" +) + +const filterLongDescription = `This command would give you a list of all the packages based on the given query` + +func setupFilterCommand() *cobraext.Command { + cmd := &cobra.Command{ + Use: "filter [flags]", + Short: "filter integrations based on given flags", + Long: filterLongDescription, + Args: cobra.NoArgs, + RunE: filterCommandAction, + } + + cmd.Flags().StringP(cobraext.FilterInputFlagName, "", "", cobraext.FilterInputFlagDescription) + cmd.Flags().StringP(cobraext.FilterCodeOwnerFlagName, "", "", cobraext.FilterCodeOwnerFlagDescription) + cmd.Flags().StringP(cobraext.FilterKibanaVersionFlagName, "", "", cobraext.FilterKibanaVersionFlagDescription) + cmd.Flags().StringP(cobraext.FilterCategoriesFlagName, "", "", cobraext.FilterCategoriesFlagDescription) + return cobraext.NewCommand(cmd, cobraext.ContextPackage) +} + +func filterCommandAction(cmd *cobra.Command, args []string) error { + cmd.Println("Filter the package") + + opts, err := fromFlags(cmd) + if err != nil { + return fmt.Errorf("getting filter options failed: %w", err) + } + + if err := opts.Validate(); err != nil { + return fmt.Errorf("validating filter options failed: %w", err) + } + + fmt.Println(opts.String()) + + return nil +} + +type filterOptions struct { + input string + codeOwner string + kibanaVersion string + categories string +} + +func fromFlags(cmd *cobra.Command) (*filterOptions, error) { + input, err := cmd.Flags().GetString(cobraext.FilterInputFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) + } + + codeOwner, err := cmd.Flags().GetString(cobraext.FilterCodeOwnerFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) + } + + kibanaVersion, err := cmd.Flags().GetString(cobraext.FilterKibanaVersionFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.FilterKibanaVersionFlagName) + } + + categories, err := cmd.Flags().GetString(cobraext.FilterCategoriesFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) + } + + return &filterOptions{ + input: input, + codeOwner: codeOwner, + kibanaVersion: kibanaVersion, + categories: categories, + }, nil +} + +func (o *filterOptions) String() string { + return fmt.Sprintf("input: %s, codeOwner: %s, kibanaVersion: %s, categories: %s", o.input, o.codeOwner, o.kibanaVersion, o.categories) +} + +func (o *filterOptions) Validate() error { + if o.input == "" && o.codeOwner == "" && o.kibanaVersion == "" && o.categories == "" { + return fmt.Errorf("at least one flag must be provided") + } + + if _, err := semver.NewConstraint(o.kibanaVersion); err != nil { + return fmt.Errorf("invalid kibana version: %w", err) + } + return nil +} diff --git a/cmd/root.go b/cmd/root.go index e449ff5169..e4a344e348 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -26,6 +26,7 @@ var commands = []*cobraext.Command{ setupDumpCommand(), setupEditCommand(), setupExportCommand(), + setupFilterCommand(), setupFormatCommand(), setupInstallCommand(), setupLinksCommand(), diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 1838404fd6..4d29613791 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -133,8 +133,21 @@ const ( FailOnMissingFlagName = "fail-on-missing" FailOnMissingFlagDescription = "fail if tests are missing" - FailFastFlagName = "fail-fast" - FailFastFlagDescription = "fail immediately if any file requires updates (do not overwrite)" + FailFastFlagName = "fail-fast" + FailFastFlagDescription = "fail immediately if any file requires updates (do not overwrite)" + + FilterInputFlagName = "input" + FilterInputFlagDescription = "name of the inputs to filter by (comma-separated values)" + + FilterCodeOwnerFlagName = "code-owner" + FilterCodeOwnerFlagDescription = "code owners to filter by (comma-separated values)" + + FilterKibanaVersionFlagName = "kibana-version" + FilterKibanaVersionFlagDescription = "kibana version to filter by (semver)" + + FilterCategoriesFlagName = "category" + FilterCategoriesFlagDescription = "integration categories to filter by (comma-separated values)" + GenerateTestResultFlagName = "generate" GenerateTestResultFlagDescription = "generate test result file" From 17e4b39c3d5b8a13586cbe59b4380e2f01534cae Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 9 Oct 2025 14:25:38 -0400 Subject: [PATCH 02/36] fix: error skipped in edit dashboard command --- cmd/edit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/edit.go b/cmd/edit.go index 38fc7d0168..ef41f80513 100644 --- a/cmd/edit.go +++ b/cmd/edit.go @@ -64,7 +64,7 @@ func editDashboardsCmd(cmd *cobra.Command, args []string) error { opts = append(opts, kibana.TLSSkipVerify()) } - allowSnapshot, _ := cmd.Flags().GetBool(cobraext.AllowSnapshotFlagName) + allowSnapshot, err := cmd.Flags().GetBool(cobraext.AllowSnapshotFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.AllowSnapshotFlagName) } From 01a681ca922539cd60b66293dfbac54e5570cea1 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 9 Oct 2025 14:25:38 -0400 Subject: [PATCH 03/36] fix: error skipped in edit dashboard command --- cmd/edit.go | 2 +- cmd/filter.go | 157 ++++++++++++++++++++++++++++++---- internal/packages/packages.go | 72 ++++++++++++++++ 3 files changed, 214 insertions(+), 17 deletions(-) diff --git a/cmd/edit.go b/cmd/edit.go index 38fc7d0168..ef41f80513 100644 --- a/cmd/edit.go +++ b/cmd/edit.go @@ -64,7 +64,7 @@ func editDashboardsCmd(cmd *cobra.Command, args []string) error { opts = append(opts, kibana.TLSSkipVerify()) } - allowSnapshot, _ := cmd.Flags().GetBool(cobraext.AllowSnapshotFlagName) + allowSnapshot, err := cmd.Flags().GetBool(cobraext.AllowSnapshotFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.AllowSnapshotFlagName) } diff --git a/cmd/filter.go b/cmd/filter.go index 7bc9ccc1ba..bb1b9e45f7 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -5,10 +5,14 @@ package cmd import ( + "errors" "fmt" + "slices" + "strings" - "github.com/Masterminds/semver/v3" "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/tui" "github.com/spf13/cobra" ) @@ -42,58 +46,179 @@ func filterCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("validating filter options failed: %w", err) } - fmt.Println(opts.String()) + pkgs, err := listPackages(opts) + if err != nil { + return fmt.Errorf("listing packages failed: %w", err) + } + + filteredPackages, err := filterPackages(opts, pkgs) + if err != nil { + return fmt.Errorf("filtering packages failed: %w", err) + } + + fmt.Print("[") + for _, pkg := range filteredPackages { + fmt.Printf("%s, ", pkg.Name) + } + fmt.Println("\b\b]") return nil } +func listPackages(opts *filterOptions) ([]packages.PackageManifest, error) { + root, found, err := packages.FindIntegrationRoot() + if err != nil { + return nil, fmt.Errorf("can't find integration root: %w", err) + } + if !found { + return nil, errors.New("integration root not found") + } + + pkgs, err := packages.ListPackages(root) + if err != nil { + return nil, fmt.Errorf("listing packages failed: %w", err) + } + + return pkgs, nil +} + type filterOptions struct { - input string - codeOwner string - kibanaVersion string - categories string + inputs []string + codeOwners []string + kibanaVersions []string + categories []string } func fromFlags(cmd *cobra.Command) (*filterOptions, error) { + opts := &filterOptions{} + input, err := cmd.Flags().GetString(cobraext.FilterInputFlagName) if err != nil { return nil, cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) } + if input != "" { + opts.inputs = strings.Split(input, ",") + } codeOwner, err := cmd.Flags().GetString(cobraext.FilterCodeOwnerFlagName) if err != nil { return nil, cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) } + if codeOwner != "" { + opts.codeOwners = strings.Split(codeOwner, ",") + } kibanaVersion, err := cmd.Flags().GetString(cobraext.FilterKibanaVersionFlagName) if err != nil { return nil, cobraext.FlagParsingError(err, cobraext.FilterKibanaVersionFlagName) } + if kibanaVersion != "" { + opts.kibanaVersions = strings.Split(kibanaVersion, ",") + } categories, err := cmd.Flags().GetString(cobraext.FilterCategoriesFlagName) if err != nil { return nil, cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) } + if categories != "" { + opts.categories = strings.Split(categories, ",") + } - return &filterOptions{ - input: input, - codeOwner: codeOwner, - kibanaVersion: kibanaVersion, - categories: categories, - }, nil + return opts, nil } func (o *filterOptions) String() string { - return fmt.Sprintf("input: %s, codeOwner: %s, kibanaVersion: %s, categories: %s", o.input, o.codeOwner, o.kibanaVersion, o.categories) + return fmt.Sprintf("input: %s, codeOwner: %s, kibanaVersion: %s, categories: %s", o.inputs, o.codeOwners, o.kibanaVersions, o.categories) } func (o *filterOptions) Validate() error { - if o.input == "" && o.codeOwner == "" && o.kibanaVersion == "" && o.categories == "" { + validator := tui.Validator{Cwd: "."} + + if len(o.inputs) == 0 && len(o.codeOwners) == 0 && len(o.kibanaVersions) == 0 && len(o.categories) == 0 { return fmt.Errorf("at least one flag must be provided") } - if _, err := semver.NewConstraint(o.kibanaVersion); err != nil { - return fmt.Errorf("invalid kibana version: %w", err) + if len(o.kibanaVersions) > 0 { + for _, version := range o.kibanaVersions { + if err := validator.Constraint(version); err != nil { + return fmt.Errorf("invalid kibana version: %w", err) + } + } + } + + if len(o.codeOwners) > 0 { + for _, owner := range o.codeOwners { + if err := validator.GithubOwner(owner); err != nil { + return fmt.Errorf("invalid code owner: %w", err) + } + } } + return nil } + +func (o *filterOptions) Filter(pkg packages.PackageManifest) bool { + codeOwner := pkg.Owner.Github + // kibanaVersion := pkg.Conditions.Kibana.Version + categories := pkg.Categories + inputs := []string{} + for _, policyTemplate := range pkg.PolicyTemplates { + if policyTemplate.Input != "" { + inputs = append(inputs, policyTemplate.Input) + } + + for _, input := range policyTemplate.Inputs { + inputs = append(inputs, input.Type) + } + } + + if len(o.inputs) > 0 { + exists := false + for _, input := range inputs { + if slices.Contains(o.inputs, input) { + exists = true + break + } + } + + if !exists { + return false + } + } + + if len(o.codeOwners) > 0 { + if !slices.Contains(o.codeOwners, codeOwner) { + return false + } + } + + if len(o.categories) > 0 { + exists := false + for _, category := range categories { + if slices.Contains(o.categories, category) { + exists = true + break + } + } + + if !exists { + return false + } + } + + return true +} + +func filterPackages(opts *filterOptions, pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) { + var filteredPackages []packages.PackageManifest + + for _, pkg := range pkgs { + if !opts.Filter(pkg) { + continue + } + + filteredPackages = append(filteredPackages, pkg) + } + + return filteredPackages, nil +} diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 3d4e60e249..dd152bb083 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -28,6 +28,9 @@ const ( // DataStreamManifestFile is the name of the data stream's manifest file. DataStreamManifestFile = "manifest.yml" + // GoModFile is the name of the go.mod file. + GoModFile = "go.mod" + defaultPipelineName = "default" dataStreamTypeLogs = "logs" @@ -328,6 +331,41 @@ func FindPackageRootFrom(fromDir string) (string, bool, error) { return "", false, nil } +// FindIntegrationRoot finds and returns the path to the root folder of a package from the working directory. +func FindIntegrationRoot() (string, bool, error) { + workDir, err := os.Getwd() + if err != nil { + return "", false, fmt.Errorf("locating working directory failed: %w", err) + } + return FindIntegrationRootFrom(workDir) +} + +// IntegrationRoot function returns the root directory of the integrations +func FindIntegrationRootFrom(fromDir string) (string, bool, error) { + rootDir := filepath.VolumeName(fromDir) + string(filepath.Separator) + + dir := fromDir + for dir != "." { + path := filepath.Join(dir, GoModFile) + fileInfo, err := os.Stat(path) + if err == nil && !fileInfo.IsDir() { + ok, err := isIntegrationRepo(path) + if err != nil { + return "", false, fmt.Errorf("verifying integration repo failed (path: %s): %w", path, err) + } + if ok { + return dir, true, nil + } + } + + if dir == rootDir { + break + } + dir = filepath.Dir(dir) + } + return fromDir, true, nil +} + // FindDataStreamRootForPath finds and returns the path to the root folder of a data stream. func FindDataStreamRootForPath(workDir string) (string, bool, error) { dir := workDir @@ -541,3 +579,37 @@ func isDataStreamManifest(path string) (bool, error) { (m.Type == dataStreamTypeLogs || m.Type == dataStreamTypeMetrics || m.Type == dataStreamTypeSynthetics || m.Type == dataStreamTypeTraces), nil } +func isIntegrationRepo(path string) (bool, error) { + modFile, err := os.ReadFile(path) + if err != nil { + return false, fmt.Errorf("reading go.mod file failed: %w", err) + } + + content := string(modFile) + content = strings.SplitN(content, "\n", 2)[0] + content = strings.TrimSpace(content) + + if !strings.HasSuffix(content, "github.com/elastic/integrations") { + return false, fmt.Errorf("integration root %s is not an elastic-package integration", path) + } + + return true, nil +} + +func ListPackages(root string) ([]PackageManifest, error) { + files, err := filepath.Glob(filepath.Join(root, "packages", "*", PackageManifestFile)) + if err != nil { + return nil, fmt.Errorf("failed matching files with package manifests: %w", err) + } + + var packages []PackageManifest + for _, file := range files { + manifest, err := ReadPackageManifest(file) + if err != nil { + return nil, fmt.Errorf("failed to read package manifest: %w", err) + } + packages = append(packages, *manifest) + } + + return packages, nil +} From 010f1a481c40926db1d5ecb84cb9a53f767dee75 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Fri, 10 Oct 2025 15:51:59 -0400 Subject: [PATCH 04/36] code refactors --- cmd/filter.go | 53 +++++++++++++++-------------------- internal/packages/packages.go | 32 +++++++++++++++++++++ 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/cmd/filter.go b/cmd/filter.go index bb1b9e45f7..344dc94683 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -5,7 +5,6 @@ package cmd import ( - "errors" "fmt" "slices" "strings" @@ -35,8 +34,6 @@ func setupFilterCommand() *cobraext.Command { } func filterCommandAction(cmd *cobra.Command, args []string) error { - cmd.Println("Filter the package") - opts, err := fromFlags(cmd) if err != nil { return fmt.Errorf("getting filter options failed: %w", err) @@ -46,40 +43,32 @@ func filterCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("validating filter options failed: %w", err) } - pkgs, err := listPackages(opts) + root, err := packages.MustFindIntegrationRoot() if err != nil { - return fmt.Errorf("listing packages failed: %w", err) + return fmt.Errorf("can't find integration root: %w", err) } - filteredPackages, err := filterPackages(opts, pkgs) + pkgs, err := packages.ReadAllPackageManifests(root) if err != nil { - return fmt.Errorf("filtering packages failed: %w", err) - } - - fmt.Print("[") - for _, pkg := range filteredPackages { - fmt.Printf("%s, ", pkg.Name) + return fmt.Errorf("listing packages failed: %w", err) } - fmt.Println("\b\b]") - - return nil -} -func listPackages(opts *filterOptions) ([]packages.PackageManifest, error) { - root, found, err := packages.FindIntegrationRoot() + filtered, err := opts.Filter(pkgs) if err != nil { - return nil, fmt.Errorf("can't find integration root: %w", err) - } - if !found { - return nil, errors.New("integration root not found") + return fmt.Errorf("filtering packages failed: %w", err) } - pkgs, err := packages.ListPackages(root) - if err != nil { - return nil, fmt.Errorf("listing packages failed: %w", err) + if len(filtered) == 0 { + return fmt.Errorf("no packages found") + } else { + fmt.Print("[") + for _, pkg := range filtered { + fmt.Printf("%s, ", pkg.Name) + } + fmt.Println("\b\b]") } - return pkgs, nil + return nil } type filterOptions struct { @@ -157,7 +146,7 @@ func (o *filterOptions) Validate() error { return nil } -func (o *filterOptions) Filter(pkg packages.PackageManifest) bool { +func (o *filterOptions) Check(pkg packages.PackageManifest) bool { codeOwner := pkg.Owner.Github // kibanaVersion := pkg.Conditions.Kibana.Version categories := pkg.Categories @@ -206,17 +195,21 @@ func (o *filterOptions) Filter(pkg packages.PackageManifest) bool { } } + // TODO: check kibana version + if len(o.kibanaVersions) > 0 { + panic("not implemented") + } + return true } -func filterPackages(opts *filterOptions, pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) { +func (o *filterOptions) Filter(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) { var filteredPackages []packages.PackageManifest for _, pkg := range pkgs { - if !opts.Filter(pkg) { + if !o.Check(pkg) { continue } - filteredPackages = append(filteredPackages, pkg) } diff --git a/internal/packages/packages.go b/internal/packages/packages.go index dd152bb083..ab73c4b73f 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -331,6 +331,19 @@ func FindPackageRootFrom(fromDir string) (string, bool, error) { return "", false, nil } +// MustFindIntegrationRoot finds and returns the path to the root folder of a package from the working directory. +// It fails with an error if the package root can't be found. +func MustFindIntegrationRoot() (string, error) { + root, found, err := FindIntegrationRoot() + if err != nil { + return "", fmt.Errorf("locating integration root failed: %w", err) + } + if !found { + return "", errors.New("integration root not found") + } + return root, nil +} + // FindIntegrationRoot finds and returns the path to the root folder of a package from the working directory. func FindIntegrationRoot() (string, bool, error) { workDir, err := os.Getwd() @@ -452,6 +465,25 @@ func ReadPackageManifest(path string) (*PackageManifest, error) { return &m, nil } +// ReadAllPackageManifests reads all the package manifests in the given root directory. +func ReadAllPackageManifests(root string) ([]PackageManifest, error) { + files, err := filepath.Glob(filepath.Join(root, "packages", "*", PackageManifestFile)) + if err != nil { + return nil, fmt.Errorf("failed matching files with package manifests: %w", err) + } + + var packages []PackageManifest + for _, file := range files { + manifest, err := ReadPackageManifest(file) + if err != nil { + return nil, fmt.Errorf("failed to read package manifest: %w", err) + } + packages = append(packages, *manifest) + } + + return packages, nil +} + // ReadTransformsFromPackageRoot looks for transforms in the given package root. func ReadTransformsFromPackageRoot(packageRoot string) ([]Transform, error) { files, err := filepath.Glob(filepath.Join(packageRoot, "elasticsearch", "transform", "*", "transform.yml")) From c3fef192a59270acbfc346e688718076b5bfdf0c Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Tue, 14 Oct 2025 14:03:38 -0400 Subject: [PATCH 05/36] update response format. --- cmd/filter.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/cmd/filter.go b/cmd/filter.go index 344dc94683..81d757887b 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -5,7 +5,10 @@ package cmd import ( + "encoding/json" "fmt" + "io" + "os" "slices" "strings" @@ -58,19 +61,29 @@ func filterCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("filtering packages failed: %w", err) } - if len(filtered) == 0 { - return fmt.Errorf("no packages found") - } else { - fmt.Print("[") - for _, pkg := range filtered { - fmt.Printf("%s, ", pkg.Name) - } - fmt.Println("\b\b]") + if err := printPkgList(filtered, os.Stdout); err != nil { + return fmt.Errorf("printing JSON failed: %w", err) } return nil } +func printPkgList(pkgs []packages.PackageManifest, w io.Writer) error { + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + + if len(pkgs) == 0 { + return nil + } + + names := []string{} + for _, pkg := range pkgs { + names = append(names, pkg.Name) + } + + return enc.Encode(names) +} + type filterOptions struct { inputs []string codeOwners []string From 3d84151844ad53a3b858fa829a6e11577bb63489 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Tue, 14 Oct 2025 16:58:11 -0400 Subject: [PATCH 06/36] move filter code to internal --- cmd/filter.go | 163 ++------------------------------ internal/filter/filter.go | 194 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+), 157 deletions(-) create mode 100644 internal/filter/filter.go diff --git a/cmd/filter.go b/cmd/filter.go index 81d757887b..b39c5f368f 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -9,12 +9,10 @@ import ( "fmt" "io" "os" - "slices" - "strings" "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/filter" "github.com/elastic/elastic-package/internal/packages" - "github.com/elastic/elastic-package/internal/tui" "github.com/spf13/cobra" ) @@ -29,20 +27,17 @@ func setupFilterCommand() *cobraext.Command { RunE: filterCommandAction, } - cmd.Flags().StringP(cobraext.FilterInputFlagName, "", "", cobraext.FilterInputFlagDescription) - cmd.Flags().StringP(cobraext.FilterCodeOwnerFlagName, "", "", cobraext.FilterCodeOwnerFlagDescription) - cmd.Flags().StringP(cobraext.FilterKibanaVersionFlagName, "", "", cobraext.FilterKibanaVersionFlagDescription) - cmd.Flags().StringP(cobraext.FilterCategoriesFlagName, "", "", cobraext.FilterCategoriesFlagDescription) + filter.SetFilterFlags(cmd) return cobraext.NewCommand(cmd, cobraext.ContextPackage) } func filterCommandAction(cmd *cobra.Command, args []string) error { - opts, err := fromFlags(cmd) + filters, err := filter.Parse(cmd) if err != nil { return fmt.Errorf("getting filter options failed: %w", err) } - if err := opts.Validate(); err != nil { + if err := filters.Validate(); err != nil { return fmt.Errorf("validating filter options failed: %w", err) } @@ -56,7 +51,7 @@ func filterCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("listing packages failed: %w", err) } - filtered, err := opts.Filter(pkgs) + filtered, err := filters.ApplyTo(pkgs) if err != nil { return fmt.Errorf("filtering packages failed: %w", err) } @@ -71,160 +66,14 @@ func filterCommandAction(cmd *cobra.Command, args []string) error { func printPkgList(pkgs []packages.PackageManifest, w io.Writer) error { enc := json.NewEncoder(w) enc.SetEscapeHTML(false) - if len(pkgs) == 0 { return nil } - names := []string{} + names := make([]string, 0, len(pkgs)) for _, pkg := range pkgs { names = append(names, pkg.Name) } return enc.Encode(names) } - -type filterOptions struct { - inputs []string - codeOwners []string - kibanaVersions []string - categories []string -} - -func fromFlags(cmd *cobra.Command) (*filterOptions, error) { - opts := &filterOptions{} - - input, err := cmd.Flags().GetString(cobraext.FilterInputFlagName) - if err != nil { - return nil, cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) - } - if input != "" { - opts.inputs = strings.Split(input, ",") - } - - codeOwner, err := cmd.Flags().GetString(cobraext.FilterCodeOwnerFlagName) - if err != nil { - return nil, cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) - } - if codeOwner != "" { - opts.codeOwners = strings.Split(codeOwner, ",") - } - - kibanaVersion, err := cmd.Flags().GetString(cobraext.FilterKibanaVersionFlagName) - if err != nil { - return nil, cobraext.FlagParsingError(err, cobraext.FilterKibanaVersionFlagName) - } - if kibanaVersion != "" { - opts.kibanaVersions = strings.Split(kibanaVersion, ",") - } - - categories, err := cmd.Flags().GetString(cobraext.FilterCategoriesFlagName) - if err != nil { - return nil, cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) - } - if categories != "" { - opts.categories = strings.Split(categories, ",") - } - - return opts, nil -} - -func (o *filterOptions) String() string { - return fmt.Sprintf("input: %s, codeOwner: %s, kibanaVersion: %s, categories: %s", o.inputs, o.codeOwners, o.kibanaVersions, o.categories) -} - -func (o *filterOptions) Validate() error { - validator := tui.Validator{Cwd: "."} - - if len(o.inputs) == 0 && len(o.codeOwners) == 0 && len(o.kibanaVersions) == 0 && len(o.categories) == 0 { - return fmt.Errorf("at least one flag must be provided") - } - - if len(o.kibanaVersions) > 0 { - for _, version := range o.kibanaVersions { - if err := validator.Constraint(version); err != nil { - return fmt.Errorf("invalid kibana version: %w", err) - } - } - } - - if len(o.codeOwners) > 0 { - for _, owner := range o.codeOwners { - if err := validator.GithubOwner(owner); err != nil { - return fmt.Errorf("invalid code owner: %w", err) - } - } - } - - return nil -} - -func (o *filterOptions) Check(pkg packages.PackageManifest) bool { - codeOwner := pkg.Owner.Github - // kibanaVersion := pkg.Conditions.Kibana.Version - categories := pkg.Categories - inputs := []string{} - for _, policyTemplate := range pkg.PolicyTemplates { - if policyTemplate.Input != "" { - inputs = append(inputs, policyTemplate.Input) - } - - for _, input := range policyTemplate.Inputs { - inputs = append(inputs, input.Type) - } - } - - if len(o.inputs) > 0 { - exists := false - for _, input := range inputs { - if slices.Contains(o.inputs, input) { - exists = true - break - } - } - - if !exists { - return false - } - } - - if len(o.codeOwners) > 0 { - if !slices.Contains(o.codeOwners, codeOwner) { - return false - } - } - - if len(o.categories) > 0 { - exists := false - for _, category := range categories { - if slices.Contains(o.categories, category) { - exists = true - break - } - } - - if !exists { - return false - } - } - - // TODO: check kibana version - if len(o.kibanaVersions) > 0 { - panic("not implemented") - } - - return true -} - -func (o *filterOptions) Filter(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) { - var filteredPackages []packages.PackageManifest - - for _, pkg := range pkgs { - if !o.Check(pkg) { - continue - } - filteredPackages = append(filteredPackages, pkg) - } - - return filteredPackages, nil -} diff --git a/internal/filter/filter.go b/internal/filter/filter.go new file mode 100644 index 0000000000..188afa2014 --- /dev/null +++ b/internal/filter/filter.go @@ -0,0 +1,194 @@ +package filter + +import ( + "fmt" + "strings" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/tui" + "github.com/spf13/cobra" +) + +type Filter struct { + Inputs map[string]struct{} + CodeOwners map[string]struct{} + KibanaVersions map[string]struct{} + Categories map[string]struct{} +} + +func NewFilter() *Filter { + return &Filter{ + Inputs: make(map[string]struct{}), + CodeOwners: make(map[string]struct{}), + KibanaVersions: make(map[string]struct{}), + Categories: make(map[string]struct{}), + } +} + +// splitAndTrim splits a string by delimiter and trims whitespace from each element +func splitAndTrim(s, delimiter string) map[string]struct{} { + if s == "" { + return nil + } + parts := strings.Split(s, delimiter) + result := make(map[string]struct{}, len(parts)) + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + result[trimmed] = struct{}{} + } + } + return result +} + +// hasAnyMatch checks if any item in the items slice exists in the filters slice +func hasAnyMatch(filters map[string]struct{}, items []string) bool { + if len(filters) == 0 { + return true + } + + for _, item := range items { + if _, ok := filters[item]; ok { + return true + } + } + + return false +} + +func SetFilterFlags(cmd *cobra.Command) { + cmd.Flags().StringP(cobraext.FilterInputFlagName, "", "", cobraext.FilterInputFlagDescription) + cmd.Flags().StringP(cobraext.FilterCodeOwnerFlagName, "", "", cobraext.FilterCodeOwnerFlagDescription) + cmd.Flags().StringP(cobraext.FilterKibanaVersionFlagName, "", "", cobraext.FilterKibanaVersionFlagDescription) + cmd.Flags().StringP(cobraext.FilterCategoriesFlagName, "", "", cobraext.FilterCategoriesFlagDescription) +} + +func Parse(cmd *cobra.Command) (*Filter, error) { + filter := &Filter{} + + input, err := cmd.Flags().GetString(cobraext.FilterInputFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) + } + filter.Inputs = splitAndTrim(input, ",") + + codeOwner, err := cmd.Flags().GetString(cobraext.FilterCodeOwnerFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) + } + filter.CodeOwners = splitAndTrim(codeOwner, ",") + + kibanaVersion, err := cmd.Flags().GetString(cobraext.FilterKibanaVersionFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.FilterKibanaVersionFlagName) + } + filter.KibanaVersions = splitAndTrim(kibanaVersion, ",") + + categories, err := cmd.Flags().GetString(cobraext.FilterCategoriesFlagName) + if err != nil { + return nil, cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) + } + filter.Categories = splitAndTrim(categories, ",") + + return filter, nil +} + +func (f *Filter) String() string { + return fmt.Sprintf( + "input: %s, codeOwner: %s, kibanaVersion: %s, categories: %s", + f.Inputs, f.CodeOwners, + f.KibanaVersions, f.Categories, + ) +} + +func (f *Filter) Validate() error { + validator := tui.Validator{Cwd: "."} + + if len(f.Inputs) == 0 && + len(f.CodeOwners) == 0 && + len(f.KibanaVersions) == 0 && + len(f.Categories) == 0 { + // No filters provided, return an error + return fmt.Errorf("at least one flag must be provided") + } + + if len(f.KibanaVersions) > 0 { + for version := range f.KibanaVersions { + if err := validator.Constraint(version); err != nil { + return fmt.Errorf("invalid kibana version: %w", err) + } + } + } + + if len(f.CodeOwners) > 0 { + for ownerName := range f.CodeOwners { + if err := validator.GithubOwner(ownerName); err != nil { + return fmt.Errorf("invalid code owner: %w", err) + } + } + } + + return nil +} + +func (f *Filter) Matches(pkg packages.PackageManifest) bool { + // Check inputs filter + if len(f.Inputs) > 0 { + inputs := f.extractInputs(pkg) + if !hasAnyMatch(f.Inputs, inputs) { + return false + } + } + + // Check code owners filter + if len(f.CodeOwners) > 0 { + if !hasAnyMatch(f.CodeOwners, []string{pkg.Owner.Github}) { + return false + } + } + + // Check categories filter + if len(f.Categories) > 0 { + if !hasAnyMatch(f.Categories, pkg.Categories) { + return false + } + } + + // Check kibana version filter + if len(f.KibanaVersions) > 0 { + // TODO: Implement kibana version filtering + // For now, return false to indicate no match + return false + } + + return true +} + +// extractInputs extracts all input types from package policy templates +func (f *Filter) extractInputs(pkg packages.PackageManifest) []string { + var inputs []string + for _, policyTemplate := range pkg.PolicyTemplates { + if policyTemplate.Input != "" { + inputs = append(inputs, policyTemplate.Input) + } + for _, input := range policyTemplate.Inputs { + inputs = append(inputs, input.Type) + } + } + return inputs +} + +func (f *Filter) ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) { + // Pre-allocate with estimated capacity to reduce reallocations + filteredPackages := make([]packages.PackageManifest, 0, len(pkgs)) + + for _, pkg := range pkgs { + if !f.Matches(pkg) { + continue + } + filteredPackages = append(filteredPackages, pkg) + } + + return filteredPackages, nil +} From 9a1c2aa760d7367d25ee4f6ad81222f288a484d6 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Tue, 14 Oct 2025 16:58:50 -0400 Subject: [PATCH 07/36] ignore .vscode dir --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4aa5e6e60d..28dd2716d3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,9 @@ elastic-package # IDEA .idea +# VSCode +.vscode + # Build directory /build From 1336ff2808c24b5113565db6b0881da6cc45c46e Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Tue, 14 Oct 2025 17:41:11 -0400 Subject: [PATCH 08/36] no args will return list of all packages --- cmd/filter.go | 7 +++++-- internal/filter/filter.go | 29 ++++++++++------------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/cmd/filter.go b/cmd/filter.go index b39c5f368f..7aeddcbeeb 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -27,13 +27,16 @@ func setupFilterCommand() *cobraext.Command { RunE: filterCommandAction, } + // add filter flags to the command (input, code owner, kibana version, categories) filter.SetFilterFlags(cmd) + return cobraext.NewCommand(cmd, cobraext.ContextPackage) } func filterCommandAction(cmd *cobra.Command, args []string) error { - filters, err := filter.Parse(cmd) - if err != nil { + filters := filter.NewFilter() + + if err := filters.Parse(cmd); err != nil { return fmt.Errorf("getting filter options failed: %w", err) } diff --git a/internal/filter/filter.go b/internal/filter/filter.go index 188afa2014..20b5489bd8 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -64,34 +64,33 @@ func SetFilterFlags(cmd *cobra.Command) { cmd.Flags().StringP(cobraext.FilterCategoriesFlagName, "", "", cobraext.FilterCategoriesFlagDescription) } -func Parse(cmd *cobra.Command) (*Filter, error) { - filter := &Filter{} +func (f *Filter) Parse(cmd *cobra.Command) error { input, err := cmd.Flags().GetString(cobraext.FilterInputFlagName) if err != nil { - return nil, cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) + return cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) } - filter.Inputs = splitAndTrim(input, ",") + f.Inputs = splitAndTrim(input, ",") codeOwner, err := cmd.Flags().GetString(cobraext.FilterCodeOwnerFlagName) if err != nil { - return nil, cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) + return cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) } - filter.CodeOwners = splitAndTrim(codeOwner, ",") + f.CodeOwners = splitAndTrim(codeOwner, ",") kibanaVersion, err := cmd.Flags().GetString(cobraext.FilterKibanaVersionFlagName) if err != nil { - return nil, cobraext.FlagParsingError(err, cobraext.FilterKibanaVersionFlagName) + return cobraext.FlagParsingError(err, cobraext.FilterKibanaVersionFlagName) } - filter.KibanaVersions = splitAndTrim(kibanaVersion, ",") + f.KibanaVersions = splitAndTrim(kibanaVersion, ",") categories, err := cmd.Flags().GetString(cobraext.FilterCategoriesFlagName) if err != nil { - return nil, cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) + return cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) } - filter.Categories = splitAndTrim(categories, ",") + f.Categories = splitAndTrim(categories, ",") - return filter, nil + return nil } func (f *Filter) String() string { @@ -105,14 +104,6 @@ func (f *Filter) String() string { func (f *Filter) Validate() error { validator := tui.Validator{Cwd: "."} - if len(f.Inputs) == 0 && - len(f.CodeOwners) == 0 && - len(f.KibanaVersions) == 0 && - len(f.Categories) == 0 { - // No filters provided, return an error - return fmt.Errorf("at least one flag must be provided") - } - if len(f.KibanaVersions) > 0 { for version := range f.KibanaVersions { if err := validator.Constraint(version); err != nil { From 53bd4f80aa8b17e8facd81b9ca19d665dd28b869 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Wed, 15 Oct 2025 18:57:40 -0400 Subject: [PATCH 09/36] initial implementation for foreach. --- cmd/filter.go | 39 +++++++++++++++-------------- cmd/foreach.go | 51 ++++++++++++++++++++++++++++++++++++++ cmd/root.go | 1 + internal/cobraext/flags.go | 3 +++ internal/filter/filter.go | 22 ++++++++++++++++ 5 files changed, 97 insertions(+), 19 deletions(-) create mode 100644 cmd/foreach.go diff --git a/cmd/filter.go b/cmd/filter.go index 7aeddcbeeb..61b77bdf0c 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -34,36 +34,37 @@ func setupFilterCommand() *cobraext.Command { } func filterCommandAction(cmd *cobra.Command, args []string) error { - filters := filter.NewFilter() - - if err := filters.Parse(cmd); err != nil { - return fmt.Errorf("getting filter options failed: %w", err) + filtered, err := filterPackage(cmd) + if err != nil { + return fmt.Errorf("filtering packages failed: %w", err) } - if err := filters.Validate(); err != nil { - return fmt.Errorf("validating filter options failed: %w", err) + if err = printPkgList(filtered, os.Stdout); err != nil { + return fmt.Errorf("printing JSON failed: %w", err) } - root, err := packages.MustFindIntegrationRoot() - if err != nil { - return fmt.Errorf("can't find integration root: %w", err) - } + return nil +} - pkgs, err := packages.ReadAllPackageManifests(root) - if err != nil { - return fmt.Errorf("listing packages failed: %w", err) +func filterPackage(cmd *cobra.Command) ([]packages.PackageManifest, error) { + var filtered []packages.PackageManifest + var err error + + filters := filter.NewFilter() + + if err = filters.Parse(cmd); err != nil { + return nil, fmt.Errorf("getting filter options failed: %w", err) } - filtered, err := filters.ApplyTo(pkgs) - if err != nil { - return fmt.Errorf("filtering packages failed: %w", err) + if err = filters.Validate(); err != nil { + return nil, fmt.Errorf("validating filter options failed: %w", err) } - if err := printPkgList(filtered, os.Stdout); err != nil { - return fmt.Errorf("printing JSON failed: %w", err) + if filtered, err = filters.Execute(); err != nil { + return nil, fmt.Errorf("filtering packages failed: %w", err) } - return nil + return filtered, nil } func printPkgList(pkgs []packages.PackageManifest, w io.Writer) error { diff --git a/cmd/foreach.go b/cmd/foreach.go new file mode 100644 index 0000000000..bdddb1ee42 --- /dev/null +++ b/cmd/foreach.go @@ -0,0 +1,51 @@ +package cmd + +import ( + "fmt" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/filter" + "github.com/spf13/cobra" +) + +const foreachLongDescription = `Execute a command for each package matching the given filter criteria. + +This command combines filtering capabilities with command execution, allowing you to run +any elastic-package subcommand across multiple packages in a single operation. + +The command uses the same filter flags as the 'filter' command (--input, --code-owner, +--kibana-version, --category) to select packages, then executes the specified subcommand +for each matched package.` + +func setupForeachCommand() *cobraext.Command { + cmd := &cobra.Command{ + Use: "foreach [filter-flags] --exec [subcommand-flags]", + Short: "Execute a command for filtered packages", + Long: foreachLongDescription, + Example: ` # Run system tests for packages with specific inputs + elastic-package foreach --input tcp,udp --exec test system -g`, + RunE: foreachCommandAction, + } + + filter.SetFilterFlags(cmd) + + // Why are we even using flags??? + // why don't we just use args instead? + // pass args to the execute command + + // cmd.Flags().StringSlice(cobraext.ForeachExecFlagName, nil, cobraext.ForeachExecFlagDescription) + // cmd.MarkFlagRequired(cobraext.ForeachExecFlagName) + + return cobraext.NewCommand(cmd, cobraext.ContextPackage) +} + +func foreachCommandAction(cmd *cobra.Command, args []string) error { + // reuse filterPackage from cmd/filter.go + filtered, err := filterPackage(cmd) + if err != nil { + return fmt.Errorf("filtering packages failed: %w", err) + } + fmt.Printf("Found %d matching package(s)\n", len(filtered)) + + return nil +} diff --git a/cmd/root.go b/cmd/root.go index e4a344e348..ae5ffe2d99 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -28,6 +28,7 @@ var commands = []*cobraext.Command{ setupExportCommand(), setupFilterCommand(), setupFormatCommand(), + setupForeachCommand(), setupInstallCommand(), setupLinksCommand(), setupLintCommand(), diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 4d29613791..95bb7bcc6c 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -148,6 +148,9 @@ const ( FilterCategoriesFlagName = "category" FilterCategoriesFlagDescription = "integration categories to filter by (comma-separated values)" + ForeachExecFlagName = "exec" + ForeachExecFlagDescription = "command to execute for each package (required)" + GenerateTestResultFlagName = "generate" GenerateTestResultFlagDescription = "generate test result file" diff --git a/internal/filter/filter.go b/internal/filter/filter.go index 20b5489bd8..fe98d92fb8 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -183,3 +183,25 @@ func (f *Filter) ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageMan return filteredPackages, nil } + +func (f *Filter) Execute() ([]packages.PackageManifest, error) { + var pkgs []packages.PackageManifest + var err error + + // Find integrations repository root + root, err := packages.MustFindIntegrationRoot() + if err != nil { + return nil, fmt.Errorf("can't find integration root: %w", err) + } + + if pkgs, err = packages.ReadAllPackageManifests(root); err != nil { + return nil, fmt.Errorf("listing packages failed: %w", err) + } + + // Apply filters and return filtered packages + if pkgs, err = f.ApplyTo(pkgs); err != nil { + return nil, fmt.Errorf("filtering packages failed: %w", err) + } + + return pkgs, nil +} From 2d827cc33c99c4d39b0579f77585ddb22deaf139 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Wed, 15 Oct 2025 19:28:21 -0400 Subject: [PATCH 10/36] run multiple commands --- cmd/foreach.go | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/cmd/foreach.go b/cmd/foreach.go index bdddb1ee42..9e38ea371c 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -2,9 +2,11 @@ package cmd import ( "fmt" + "path/filepath" "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/filter" + "github.com/elastic/elastic-package/internal/packages" "github.com/spf13/cobra" ) @@ -29,17 +31,16 @@ func setupForeachCommand() *cobraext.Command { filter.SetFilterFlags(cmd) - // Why are we even using flags??? - // why don't we just use args instead? - // pass args to the execute command - - // cmd.Flags().StringSlice(cobraext.ForeachExecFlagName, nil, cobraext.ForeachExecFlagDescription) - // cmd.MarkFlagRequired(cobraext.ForeachExecFlagName) - return cobraext.NewCommand(cmd, cobraext.ContextPackage) } func foreachCommandAction(cmd *cobra.Command, args []string) error { + // Find integration root + root, err := packages.MustFindIntegrationRoot() + if err != nil { + return fmt.Errorf("can't find integration root: %w", err) + } + // reuse filterPackage from cmd/filter.go filtered, err := filterPackage(cmd) if err != nil { @@ -47,5 +48,21 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { } fmt.Printf("Found %d matching package(s)\n", len(filtered)) + // Execute command for each package + for _, pkg := range filtered { + // Get elastic-package command + ep := cmd.Parent() + + // Set change directory flag to the package directory + ep.Flags().Set(cobraext.ChangeDirectoryFlagName, filepath.Join(root, "packages", pkg.Name)) + + ep.SetArgs(args) + + // Execute command + if err := ep.Execute(); err != nil { + return fmt.Errorf("executing command for package %s failed: %w", pkg.Name, err) + } + } + return nil } From bc7a1672b7137575d8c040a761f6accb5e8d31d9 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Wed, 15 Oct 2025 19:44:54 -0400 Subject: [PATCH 11/36] add option to run commands in parallel --- cmd/foreach.go | 60 +++++++++++++++++++++++++++++++------- internal/cobraext/flags.go | 3 ++ 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/cmd/foreach.go b/cmd/foreach.go index 9e38ea371c..ed3edd6546 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -3,9 +3,11 @@ package cmd import ( "fmt" "path/filepath" + "sync" "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/filter" + "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" "github.com/spf13/cobra" ) @@ -31,10 +33,17 @@ func setupForeachCommand() *cobraext.Command { filter.SetFilterFlags(cmd) + cmd.Flags().IntP(cobraext.ForeachPoolSizeFlagName, "p", 1, cobraext.ForeachPoolSizeFlagDescription) + return cobraext.NewCommand(cmd, cobraext.ContextPackage) } func foreachCommandAction(cmd *cobra.Command, args []string) error { + poolSize, err := cmd.Flags().GetInt(cobraext.ForeachPoolSizeFlagName) + if err != nil { + return fmt.Errorf("getting pool size failed: %w", err) + } + // Find integration root root, err := packages.MustFindIntegrationRoot() if err != nil { @@ -48,20 +57,51 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { } fmt.Printf("Found %d matching package(s)\n", len(filtered)) - // Execute command for each package + // Get elastic-package command + ep := cmd.Parent() + ep.SetArgs(args) + + wg := sync.WaitGroup{} + mu := sync.Mutex{} + errs := multierror.Error{} + + packageChan := make(chan string, poolSize) + + for range poolSize { + wg.Add(1) + go func(packageChan <-chan string) { + defer wg.Done() + for packageName := range packageChan { + if err := executeCommand(ep, args, root, packageName); err != nil { + mu.Lock() + errs = append(errs, fmt.Errorf("executing command for package %s failed: %w", packageName, err)) + mu.Unlock() + } + } + }(packageChan) + } + for _, pkg := range filtered { - // Get elastic-package command - ep := cmd.Parent() + packageChan <- pkg.Name + } + close(packageChan) - // Set change directory flag to the package directory - ep.Flags().Set(cobraext.ChangeDirectoryFlagName, filepath.Join(root, "packages", pkg.Name)) + wg.Wait() + + if errs.Error() != "" { + return fmt.Errorf("errors occurred while executing command for packages: \n%s", errs.Error()) + } + + return nil +} - ep.SetArgs(args) +func executeCommand(ep *cobra.Command, args []string, root string, packageName string) error { + // Set change directory flag to the package directory + ep.Flags().Set(cobraext.ChangeDirectoryFlagName, filepath.Join(root, "packages", packageName)) - // Execute command - if err := ep.Execute(); err != nil { - return fmt.Errorf("executing command for package %s failed: %w", pkg.Name, err) - } + // Execute command + if err := ep.Execute(); err != nil { + return fmt.Errorf("executing command for package %s failed: %w", packageName, err) } return nil diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 95bb7bcc6c..cd3123c4f4 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -151,6 +151,9 @@ const ( ForeachExecFlagName = "exec" ForeachExecFlagDescription = "command to execute for each package (required)" + ForeachPoolSizeFlagName = "parallel" + ForeachPoolSizeFlagDescription = "number of packages to execute in parallel (defaults to serial execution)" + GenerateTestResultFlagName = "generate" GenerateTestResultFlagDescription = "generate test result file" From ada16f7ba7575867adf1ae9caafeb4983eb9d92f Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 16 Oct 2025 11:46:26 -0400 Subject: [PATCH 12/36] stash: filter-registry --- internal/cobraext/flags.go | 11 +- internal/filter/filter.go | 207 ++---------------------------------- internal/filter/input.go | 59 ++++++++++ internal/filter/registry.go | 49 +++++++++ internal/filter/utils.go | 34 ++++++ 5 files changed, 160 insertions(+), 200 deletions(-) create mode 100644 internal/filter/input.go create mode 100644 internal/filter/registry.go create mode 100644 internal/filter/utils.go diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index cd3123c4f4..62f0a8c29c 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -136,17 +136,20 @@ const ( FailFastFlagName = "fail-fast" FailFastFlagDescription = "fail immediately if any file requires updates (do not overwrite)" - FilterInputFlagName = "input" - FilterInputFlagDescription = "name of the inputs to filter by (comma-separated values)" + FilterCategoriesFlagName = "category" + FilterCategoriesFlagDescription = "integration categories to filter by (comma-separated values)" FilterCodeOwnerFlagName = "code-owner" FilterCodeOwnerFlagDescription = "code owners to filter by (comma-separated values)" + FilterInputFlagName = "input" + FilterInputFlagDescription = "name of the inputs to filter by (comma-separated values)" + FilterKibanaVersionFlagName = "kibana-version" FilterKibanaVersionFlagDescription = "kibana version to filter by (semver)" - FilterCategoriesFlagName = "category" - FilterCategoriesFlagDescription = "integration categories to filter by (comma-separated values)" + FilterPackagesFlagName = "packages" + FilterPackagesFlagDescription = "packages to filter by (glob pattern/comma-separated values)" ForeachExecFlagName = "exec" ForeachExecFlagDescription = "command to execute for each package (required)" diff --git a/internal/filter/filter.go b/internal/filter/filter.go index fe98d92fb8..2226657b9d 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -1,207 +1,22 @@ package filter import ( - "fmt" - "strings" - - "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/packages" - "github.com/elastic/elastic-package/internal/tui" "github.com/spf13/cobra" ) -type Filter struct { - Inputs map[string]struct{} - CodeOwners map[string]struct{} - KibanaVersions map[string]struct{} - Categories map[string]struct{} -} - -func NewFilter() *Filter { - return &Filter{ - Inputs: make(map[string]struct{}), - CodeOwners: make(map[string]struct{}), - KibanaVersions: make(map[string]struct{}), - Categories: make(map[string]struct{}), - } -} - -// splitAndTrim splits a string by delimiter and trims whitespace from each element -func splitAndTrim(s, delimiter string) map[string]struct{} { - if s == "" { - return nil - } - parts := strings.Split(s, delimiter) - result := make(map[string]struct{}, len(parts)) - for _, part := range parts { - trimmed := strings.TrimSpace(part) - if trimmed != "" { - result[trimmed] = struct{}{} - } - } - return result -} - -// hasAnyMatch checks if any item in the items slice exists in the filters slice -func hasAnyMatch(filters map[string]struct{}, items []string) bool { - if len(filters) == 0 { - return true - } - - for _, item := range items { - if _, ok := filters[item]; ok { - return true - } - } - - return false -} - -func SetFilterFlags(cmd *cobra.Command) { - cmd.Flags().StringP(cobraext.FilterInputFlagName, "", "", cobraext.FilterInputFlagDescription) - cmd.Flags().StringP(cobraext.FilterCodeOwnerFlagName, "", "", cobraext.FilterCodeOwnerFlagDescription) - cmd.Flags().StringP(cobraext.FilterKibanaVersionFlagName, "", "", cobraext.FilterKibanaVersionFlagDescription) - cmd.Flags().StringP(cobraext.FilterCategoriesFlagName, "", "", cobraext.FilterCategoriesFlagDescription) -} - -func (f *Filter) Parse(cmd *cobra.Command) error { - - input, err := cmd.Flags().GetString(cobraext.FilterInputFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) - } - f.Inputs = splitAndTrim(input, ",") - - codeOwner, err := cmd.Flags().GetString(cobraext.FilterCodeOwnerFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) - } - f.CodeOwners = splitAndTrim(codeOwner, ",") - - kibanaVersion, err := cmd.Flags().GetString(cobraext.FilterKibanaVersionFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.FilterKibanaVersionFlagName) - } - f.KibanaVersions = splitAndTrim(kibanaVersion, ",") - - categories, err := cmd.Flags().GetString(cobraext.FilterCategoriesFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) - } - f.Categories = splitAndTrim(categories, ",") - - return nil -} - -func (f *Filter) String() string { - return fmt.Sprintf( - "input: %s, codeOwner: %s, kibanaVersion: %s, categories: %s", - f.Inputs, f.CodeOwners, - f.KibanaVersions, f.Categories, - ) +type FilterFlagImpl interface { + Name() string + Description() string + Shorthand() string + DefaultValue() string } -func (f *Filter) Validate() error { - validator := tui.Validator{Cwd: "."} - - if len(f.KibanaVersions) > 0 { - for version := range f.KibanaVersions { - if err := validator.Constraint(version); err != nil { - return fmt.Errorf("invalid kibana version: %w", err) - } - } - } - - if len(f.CodeOwners) > 0 { - for ownerName := range f.CodeOwners { - if err := validator.GithubOwner(ownerName); err != nil { - return fmt.Errorf("invalid code owner: %w", err) - } - } - } - - return nil -} - -func (f *Filter) Matches(pkg packages.PackageManifest) bool { - // Check inputs filter - if len(f.Inputs) > 0 { - inputs := f.extractInputs(pkg) - if !hasAnyMatch(f.Inputs, inputs) { - return false - } - } - - // Check code owners filter - if len(f.CodeOwners) > 0 { - if !hasAnyMatch(f.CodeOwners, []string{pkg.Owner.Github}) { - return false - } - } - - // Check categories filter - if len(f.Categories) > 0 { - if !hasAnyMatch(f.Categories, pkg.Categories) { - return false - } - } - - // Check kibana version filter - if len(f.KibanaVersions) > 0 { - // TODO: Implement kibana version filtering - // For now, return false to indicate no match - return false - } - - return true -} - -// extractInputs extracts all input types from package policy templates -func (f *Filter) extractInputs(pkg packages.PackageManifest) []string { - var inputs []string - for _, policyTemplate := range pkg.PolicyTemplates { - if policyTemplate.Input != "" { - inputs = append(inputs, policyTemplate.Input) - } - for _, input := range policyTemplate.Inputs { - inputs = append(inputs, input.Type) - } - } - return inputs -} - -func (f *Filter) ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) { - // Pre-allocate with estimated capacity to reduce reallocations - filteredPackages := make([]packages.PackageManifest, 0, len(pkgs)) - - for _, pkg := range pkgs { - if !f.Matches(pkg) { - continue - } - filteredPackages = append(filteredPackages, pkg) - } - - return filteredPackages, nil -} - -func (f *Filter) Execute() ([]packages.PackageManifest, error) { - var pkgs []packages.PackageManifest - var err error - - // Find integrations repository root - root, err := packages.MustFindIntegrationRoot() - if err != nil { - return nil, fmt.Errorf("can't find integration root: %w", err) - } - - if pkgs, err = packages.ReadAllPackageManifests(root); err != nil { - return nil, fmt.Errorf("listing packages failed: %w", err) - } - - // Apply filters and return filtered packages - if pkgs, err = f.ApplyTo(pkgs); err != nil { - return nil, fmt.Errorf("filtering packages failed: %w", err) - } +type FilterImpl interface { + FilterFlagImpl - return pkgs, nil + Parse(cmd *cobra.Command) error + Validate() error + Matches(pkg packages.PackageManifest) bool + ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) } diff --git a/internal/filter/input.go b/internal/filter/input.go new file mode 100644 index 0000000000..ed9531d90b --- /dev/null +++ b/internal/filter/input.go @@ -0,0 +1,59 @@ +package filter + +import ( + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" + "github.com/spf13/cobra" +) + +type InputFlag struct { + name string + description string + shorthand string + defaultValue string +} + +func (f *InputFlag) Name() string { + return f.name +} + +func (f *InputFlag) Description() string { + return f.description +} + +func (f *InputFlag) Shorthand() string { + return f.shorthand +} + +func (f *InputFlag) DefaultValue() string { + return f.defaultValue +} + +func (f *InputFlag) AddToCommand(cmd *cobra.Command) *string { + return cmd.Flags().StringP(f.Name(), f.Shorthand(), f.DefaultValue(), f.Description()) +} + +func (f *InputFlag) Parse(cmd *cobra.Command) error { + return nil +} + +func (f *InputFlag) Validate() error { + return nil +} + +func (f *InputFlag) Matches(pkg packages.PackageManifest) bool { + return true +} + +func (f *InputFlag) ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) { + return pkgs, nil +} + +func setupInputFlag() *InputFlag { + return &InputFlag{ + name: cobraext.FilterInputFlagName, + description: cobraext.FilterInputFlagDescription, + shorthand: "", + defaultValue: "", + } +} diff --git a/internal/filter/registry.go b/internal/filter/registry.go new file mode 100644 index 0000000000..904cf2dd49 --- /dev/null +++ b/internal/filter/registry.go @@ -0,0 +1,49 @@ +package filter + +import ( + "github.com/elastic/elastic-package/internal/packages" + "github.com/spf13/cobra" +) + +var filterRegistry *FilterRegistry = NewFilterRegistry() + +func init() { + filterRegistry.Register(setupInputFlag()) +} + +func SetFilterFlags(cmd *cobra.Command) { + for _, filter := range filterRegistry.filters { + cmd.Flags().StringP(filter.Name(), filter.Shorthand(), filter.DefaultValue(), filter.Description()) + } +} + +type FilterRegistry struct { + filters []FilterImpl +} + +func NewFilterRegistry() *FilterRegistry { + return &FilterRegistry{ + filters: make([]FilterImpl, 0), + } +} + +func (r *FilterRegistry) Register(filter FilterImpl) { + r.filters = append(r.filters, filter) +} + +func (r *FilterRegistry) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { + filtered = pkgs + + for _, filter := range r.filters { + filtered, err = filter.ApplyTo(filtered) + if err != nil { + return nil, err + } + } + + return filtered, nil +} + +func (r *FilterRegistry) Execute(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { + return r.ApplyTo(pkgs) +} diff --git a/internal/filter/utils.go b/internal/filter/utils.go new file mode 100644 index 0000000000..a39915eee7 --- /dev/null +++ b/internal/filter/utils.go @@ -0,0 +1,34 @@ +package filter + +import "strings" + +// splitAndTrim splits a string by delimiter and trims whitespace from each element +func splitAndTrim(s, delimiter string) map[string]struct{} { + if s == "" { + return nil + } + parts := strings.Split(s, delimiter) + result := make(map[string]struct{}, len(parts)) + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + result[trimmed] = struct{}{} + } + } + return result +} + +// hasAnyMatch checks if any item in the items slice exists in the filters slice +func hasAnyMatch(filters map[string]struct{}, items []string) bool { + if len(filters) == 0 { + return true + } + + for _, item := range items { + if _, ok := filters[item]; ok { + return true + } + } + + return false +} From 9123daeff2ae534d39bdc829cef6f99940942901 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 16 Oct 2025 13:31:40 -0400 Subject: [PATCH 13/36] working with subcommand flags --- cmd/foreach.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/cmd/foreach.go b/cmd/foreach.go index ed3edd6546..46a9021e3a 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "path/filepath" + "strings" "sync" "github.com/elastic/elastic-package/internal/cobraext" @@ -17,28 +18,39 @@ const foreachLongDescription = `Execute a command for each package matching the This command combines filtering capabilities with command execution, allowing you to run any elastic-package subcommand across multiple packages in a single operation. -The command uses the same filter flags as the 'filter' command (--input, --code-owner, ---kibana-version, --category) to select packages, then executes the specified subcommand -for each matched package.` +The command uses the same filter flags as the 'filter' command to select packages, +then executes the specified subcommand for each matched package.` func setupForeachCommand() *cobraext.Command { cmd := &cobra.Command{ - Use: "foreach [filter-flags] --exec [subcommand-flags]", + Use: "foreach --exec ''", Short: "Execute a command for filtered packages", Long: foreachLongDescription, Example: ` # Run system tests for packages with specific inputs - elastic-package foreach --input tcp,udp --exec test system -g`, + elastic-package foreach --exec 'test system -g' --input tcp,udp`, RunE: foreachCommandAction, + Args: cobra.NoArgs, } + // Add filter flags filter.SetFilterFlags(cmd) - cmd.Flags().IntP(cobraext.ForeachPoolSizeFlagName, "p", 1, cobraext.ForeachPoolSizeFlagDescription) + // Add pool size flag + cmd.Flags().IntP(cobraext.ForeachPoolSizeFlagName, "", 1, cobraext.ForeachPoolSizeFlagDescription) + + // Add exec flag and mark it as required + cmd.Flags().StringP(cobraext.ForeachExecFlagName, "", "", cobraext.ForeachExecFlagDescription) + cmd.MarkFlagRequired(cobraext.ForeachExecFlagName) return cobraext.NewCommand(cmd, cobraext.ContextPackage) } func foreachCommandAction(cmd *cobra.Command, args []string) error { + exec, err := cmd.Flags().GetString(cobraext.ForeachExecFlagName) + if err != nil { + return fmt.Errorf("getting exec failed: %w", err) + } + poolSize, err := cmd.Flags().GetInt(cobraext.ForeachPoolSizeFlagName) if err != nil { return fmt.Errorf("getting pool size failed: %w", err) @@ -59,7 +71,11 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { // Get elastic-package command ep := cmd.Parent() - ep.SetArgs(args) + + // Split the exec command string into arguments and append the command arguments + execArgs := strings.Split(exec, " ") + newArgs := append(args, execArgs...) + ep.SetArgs(newArgs) wg := sync.WaitGroup{} mu := sync.Mutex{} From 2e3721c66a926e938cad0dd358fc1207ce9a40e8 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 16 Oct 2025 14:36:19 -0400 Subject: [PATCH 14/36] created structure with working input filter --- cmd/filter.go | 2 +- internal/filter/filter.go | 9 +----- internal/filter/input.go | 44 +++++++++++++++------------- internal/filter/registry.go | 57 +++++++++++++++++++++++++++++++------ internal/filter/utils.go | 27 +++++++++++++++++- 5 files changed, 101 insertions(+), 38 deletions(-) diff --git a/cmd/filter.go b/cmd/filter.go index 61b77bdf0c..76a2f42aae 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -50,7 +50,7 @@ func filterPackage(cmd *cobra.Command) ([]packages.PackageManifest, error) { var filtered []packages.PackageManifest var err error - filters := filter.NewFilter() + filters := filter.NewFilterRegistry() if err = filters.Parse(cmd); err != nil { return nil, fmt.Errorf("getting filter options failed: %w", err) diff --git a/internal/filter/filter.go b/internal/filter/filter.go index 2226657b9d..67ca26d341 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -5,15 +5,8 @@ import ( "github.com/spf13/cobra" ) -type FilterFlagImpl interface { - Name() string - Description() string - Shorthand() string - DefaultValue() string -} - type FilterImpl interface { - FilterFlagImpl + Register(cmd *cobra.Command) Parse(cmd *cobra.Command) error Validate() error diff --git a/internal/filter/input.go b/internal/filter/input.go index ed9531d90b..ac7aa1b3ff 100644 --- a/internal/filter/input.go +++ b/internal/filter/input.go @@ -11,29 +11,21 @@ type InputFlag struct { description string shorthand string defaultValue string -} - -func (f *InputFlag) Name() string { - return f.name -} - -func (f *InputFlag) Description() string { - return f.description -} -func (f *InputFlag) Shorthand() string { - return f.shorthand + // flag specific fields + inputs map[string]struct{} } -func (f *InputFlag) DefaultValue() string { - return f.defaultValue -} - -func (f *InputFlag) AddToCommand(cmd *cobra.Command) *string { - return cmd.Flags().StringP(f.Name(), f.Shorthand(), f.DefaultValue(), f.Description()) +func (f *InputFlag) Register(cmd *cobra.Command) { + cmd.Flags().StringP(f.name, f.shorthand, f.defaultValue, f.description) } func (f *InputFlag) Parse(cmd *cobra.Command) error { + input, err := cmd.Flags().GetString(cobraext.FilterInputFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) + } + f.inputs = splitAndTrim(input, ",") return nil } @@ -42,14 +34,26 @@ func (f *InputFlag) Validate() error { } func (f *InputFlag) Matches(pkg packages.PackageManifest) bool { + if f.inputs != nil { + inputs := extractInputs(pkg) + if !hasAnyMatch(f.inputs, inputs) { + return false + } + } return true } -func (f *InputFlag) ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) { - return pkgs, nil +func (f *InputFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { + for _, pkg := range pkgs { + if !f.Matches(pkg) { + continue + } + filtered = append(filtered, pkg) + } + return filtered, nil } -func setupInputFlag() *InputFlag { +func initInputFlag() *InputFlag { return &InputFlag{ name: cobraext.FilterInputFlagName, description: cobraext.FilterInputFlagDescription, diff --git a/internal/filter/registry.go b/internal/filter/registry.go index 904cf2dd49..28658ecabc 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -1,19 +1,20 @@ package filter import ( + "fmt" + + "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" "github.com/spf13/cobra" ) -var filterRegistry *FilterRegistry = NewFilterRegistry() - -func init() { - filterRegistry.Register(setupInputFlag()) +var registry = []FilterImpl{ + initInputFlag(), } func SetFilterFlags(cmd *cobra.Command) { - for _, filter := range filterRegistry.filters { - cmd.Flags().StringP(filter.Name(), filter.Shorthand(), filter.DefaultValue(), filter.Description()) + for _, flag := range registry { + flag.Register(cmd) } } @@ -23,8 +24,32 @@ type FilterRegistry struct { func NewFilterRegistry() *FilterRegistry { return &FilterRegistry{ - filters: make([]FilterImpl, 0), + filters: registry, + } +} + +func (r *FilterRegistry) Parse(cmd *cobra.Command) error { + errs := multierror.Error{} + for _, filter := range r.filters { + if err := filter.Parse(cmd); err != nil { + errs = append(errs, err) + } + } + + if errs.Error() != "" { + return fmt.Errorf("error parsing filter options: %s", errs.Error()) + } + + return nil +} + +func (r *FilterRegistry) Validate() error { + for _, filter := range r.filters { + if err := filter.Validate(); err != nil { + return err + } } + return nil } func (r *FilterRegistry) Register(filter FilterImpl) { @@ -39,11 +64,27 @@ func (r *FilterRegistry) ApplyTo(pkgs []packages.PackageManifest) (filtered []pa if err != nil { return nil, err } + + if filtered == nil { + return nil, nil + } + + fmt.Printf("Filtered %d packages\n", len(filtered)) } return filtered, nil } -func (r *FilterRegistry) Execute(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { +func (r *FilterRegistry) Execute() (filtered []packages.PackageManifest, err error) { + root, err := packages.MustFindIntegrationRoot() + if err != nil { + return nil, err + } + + pkgs, err := packages.ReadAllPackageManifests(root) + if err != nil { + return nil, err + } + return r.ApplyTo(pkgs) } diff --git a/internal/filter/utils.go b/internal/filter/utils.go index a39915eee7..eb6163c3f0 100644 --- a/internal/filter/utils.go +++ b/internal/filter/utils.go @@ -1,6 +1,10 @@ package filter -import "strings" +import ( + "strings" + + "github.com/elastic/elastic-package/internal/packages" +) // splitAndTrim splits a string by delimiter and trims whitespace from each element func splitAndTrim(s, delimiter string) map[string]struct{} { @@ -32,3 +36,24 @@ func hasAnyMatch(filters map[string]struct{}, items []string) bool { return false } + +// extractInputs extracts all input types from package policy templates +func extractInputs(pkg packages.PackageManifest) []string { + var inputs []string + uniqueInputs := make(map[string]struct{}) + for _, policyTemplate := range pkg.PolicyTemplates { + if policyTemplate.Input != "" { + uniqueInputs[policyTemplate.Input] = struct{}{} + } + + for _, input := range policyTemplate.Inputs { + uniqueInputs[input.Type] = struct{}{} + } + } + + for input := range uniqueInputs { + inputs = append(inputs, input) + } + + return inputs +} From 8180ff4b2a483dacc723f2c66272fdfdc03dae7c Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 16 Oct 2025 15:06:10 -0400 Subject: [PATCH 15/36] minor refactor; add code owner flag --- internal/filter/codeowner.go | 73 ++++++++++++++++++++++++++++++++++++ internal/filter/input.go | 5 +-- internal/filter/registry.go | 36 ++++++------------ 3 files changed, 87 insertions(+), 27 deletions(-) create mode 100644 internal/filter/codeowner.go diff --git a/internal/filter/codeowner.go b/internal/filter/codeowner.go new file mode 100644 index 0000000000..41d8f7a0d5 --- /dev/null +++ b/internal/filter/codeowner.go @@ -0,0 +1,73 @@ +package filter + +import ( + "fmt" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/tui" + "github.com/spf13/cobra" +) + +type CodeOwnerFlag struct { + name string + description string + shorthand string + defaultValue string + + values map[string]struct{} +} + +func (f *CodeOwnerFlag) Register(cmd *cobra.Command) { + cmd.Flags().StringP(f.name, f.shorthand, f.defaultValue, f.description) +} + +func (f *CodeOwnerFlag) Parse(cmd *cobra.Command) error { + codeOwners, err := cmd.Flags().GetString(cobraext.FilterCodeOwnerFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) + } + f.values = splitAndTrim(codeOwners, ",") + + if err := f.Validate(); err != nil { + return err + } + + return nil +} + +func (f *CodeOwnerFlag) Validate() error { + validator := tui.Validator{Cwd: "."} + + if f.values != nil { + for value := range f.values { + if err := validator.GithubOwner(value); err != nil { + return fmt.Errorf("invalid code owner: %s: %w", value, err) + } + } + } + + return nil +} + +func (f *CodeOwnerFlag) Matches(pkg packages.PackageManifest) bool { + return hasAnyMatch(f.values, []string{pkg.Owner.Github}) +} + +func (f *CodeOwnerFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { + for _, pkg := range pkgs { + if f.Matches(pkg) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initCodeOwnerFlag() *CodeOwnerFlag { + return &CodeOwnerFlag{ + name: cobraext.FilterCodeOwnerFlagName, + description: cobraext.FilterCodeOwnerFlagDescription, + shorthand: "", + defaultValue: "", + } +} diff --git a/internal/filter/input.go b/internal/filter/input.go index ac7aa1b3ff..ba9a8d62fa 100644 --- a/internal/filter/input.go +++ b/internal/filter/input.go @@ -45,10 +45,9 @@ func (f *InputFlag) Matches(pkg packages.PackageManifest) bool { func (f *InputFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { for _, pkg := range pkgs { - if !f.Matches(pkg) { - continue + if f.Matches(pkg) { + filtered = append(filtered, pkg) } - filtered = append(filtered, pkg) } return filtered, nil } diff --git a/internal/filter/registry.go b/internal/filter/registry.go index 28658ecabc..b4bb04e96f 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -3,6 +3,7 @@ package filter import ( "fmt" + "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" "github.com/spf13/cobra" @@ -10,6 +11,7 @@ import ( var registry = []FilterImpl{ initInputFlag(), + initCodeOwnerFlag(), } func SetFilterFlags(cmd *cobra.Command) { @@ -52,29 +54,6 @@ func (r *FilterRegistry) Validate() error { return nil } -func (r *FilterRegistry) Register(filter FilterImpl) { - r.filters = append(r.filters, filter) -} - -func (r *FilterRegistry) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { - filtered = pkgs - - for _, filter := range r.filters { - filtered, err = filter.ApplyTo(filtered) - if err != nil { - return nil, err - } - - if filtered == nil { - return nil, nil - } - - fmt.Printf("Filtered %d packages\n", len(filtered)) - } - - return filtered, nil -} - func (r *FilterRegistry) Execute() (filtered []packages.PackageManifest, err error) { root, err := packages.MustFindIntegrationRoot() if err != nil { @@ -86,5 +65,14 @@ func (r *FilterRegistry) Execute() (filtered []packages.PackageManifest, err err return nil, err } - return r.ApplyTo(pkgs) + filtered = pkgs + for _, filter := range r.filters { + filtered, err = filter.ApplyTo(filtered) + if err != nil || len(filtered) == 0 { + break + } + } + + logger.Infof("Filtered %d packages\n", len(filtered)) + return filtered, nil } From f2abc114ecd4efdad2fabf0548c3a94c69bcdadb Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 16 Oct 2025 15:18:48 -0400 Subject: [PATCH 16/36] minor refactor --- internal/filter/filter.go | 15 --------------- internal/filter/input.go | 8 ++++---- internal/filter/registry.go | 13 +++++++++++-- 3 files changed, 15 insertions(+), 21 deletions(-) delete mode 100644 internal/filter/filter.go diff --git a/internal/filter/filter.go b/internal/filter/filter.go deleted file mode 100644 index 67ca26d341..0000000000 --- a/internal/filter/filter.go +++ /dev/null @@ -1,15 +0,0 @@ -package filter - -import ( - "github.com/elastic/elastic-package/internal/packages" - "github.com/spf13/cobra" -) - -type FilterImpl interface { - Register(cmd *cobra.Command) - - Parse(cmd *cobra.Command) error - Validate() error - Matches(pkg packages.PackageManifest) bool - ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) -} diff --git a/internal/filter/input.go b/internal/filter/input.go index ba9a8d62fa..bc8e82fe75 100644 --- a/internal/filter/input.go +++ b/internal/filter/input.go @@ -13,7 +13,7 @@ type InputFlag struct { defaultValue string // flag specific fields - inputs map[string]struct{} + values map[string]struct{} } func (f *InputFlag) Register(cmd *cobra.Command) { @@ -25,7 +25,7 @@ func (f *InputFlag) Parse(cmd *cobra.Command) error { if err != nil { return cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) } - f.inputs = splitAndTrim(input, ",") + f.values = splitAndTrim(input, ",") return nil } @@ -34,9 +34,9 @@ func (f *InputFlag) Validate() error { } func (f *InputFlag) Matches(pkg packages.PackageManifest) bool { - if f.inputs != nil { + if f.values != nil { inputs := extractInputs(pkg) - if !hasAnyMatch(f.inputs, inputs) { + if !hasAnyMatch(f.values, inputs) { return false } } diff --git a/internal/filter/registry.go b/internal/filter/registry.go index b4bb04e96f..933f2a0048 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -9,14 +9,23 @@ import ( "github.com/spf13/cobra" ) +type FilterImpl interface { + Register(cmd *cobra.Command) + + Parse(cmd *cobra.Command) error + Validate() error + Matches(pkg packages.PackageManifest) bool + ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) +} + var registry = []FilterImpl{ initInputFlag(), initCodeOwnerFlag(), } func SetFilterFlags(cmd *cobra.Command) { - for _, flag := range registry { - flag.Register(cmd) + for _, filter := range registry { + filter.Register(cmd) } } From f72ea26bb7ac99ee4b337b11bbb58f4bd5ea62d3 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 16 Oct 2025 15:42:49 -0400 Subject: [PATCH 17/36] add category flag --- internal/filter/category.go | 55 +++++++++++++++++++++++++++++++++++++ internal/filter/registry.go | 1 + 2 files changed, 56 insertions(+) create mode 100644 internal/filter/category.go diff --git a/internal/filter/category.go b/internal/filter/category.go new file mode 100644 index 0000000000..b70e19a602 --- /dev/null +++ b/internal/filter/category.go @@ -0,0 +1,55 @@ +package filter + +import ( + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" + "github.com/spf13/cobra" +) + +type CategoryFlag struct { + name string + description string + shorthand string + defaultValue string + + values map[string]struct{} +} + +func (f *CategoryFlag) Register(cmd *cobra.Command) { + cmd.Flags().StringP(f.name, f.shorthand, f.defaultValue, f.description) +} + +func (f *CategoryFlag) Parse(cmd *cobra.Command) error { + category, err := cmd.Flags().GetString(cobraext.FilterCategoriesFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) + } + f.values = splitAndTrim(category, ",") + return nil +} + +func (f *CategoryFlag) Validate() error { + return nil +} + +func (f *CategoryFlag) Matches(pkg packages.PackageManifest) bool { + return hasAnyMatch(f.values, pkg.Categories) +} + +func (f *CategoryFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { + for _, pkg := range pkgs { + if f.Matches(pkg) { + filtered = append(filtered, pkg) + } + } + return filtered, err +} + +func initCategoryFlag() *CategoryFlag { + return &CategoryFlag{ + name: cobraext.FilterCategoriesFlagName, + description: cobraext.FilterCategoriesFlagDescription, + shorthand: "", + defaultValue: "", + } +} diff --git a/internal/filter/registry.go b/internal/filter/registry.go index 933f2a0048..1194295d71 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -21,6 +21,7 @@ type FilterImpl interface { var registry = []FilterImpl{ initInputFlag(), initCodeOwnerFlag(), + initCategoryFlag(), } func SetFilterFlags(cmd *cobra.Command) { From 5c4d63e2c37874b4d0ecd740776c427bd9d26a19 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 16 Oct 2025 17:02:42 -0400 Subject: [PATCH 18/36] refactor code --- cmd/foreach.go | 5 ++-- internal/filter/category.go | 19 +++++--------- internal/filter/codeowner.go | 20 +++++--------- internal/filter/input.go | 19 +++++--------- internal/filter/registry.go | 17 +++--------- internal/filter/type.go | 51 ++++++++++++++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 52 deletions(-) create mode 100644 internal/filter/type.go diff --git a/cmd/foreach.go b/cmd/foreach.go index 46a9021e3a..5f2e404e71 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -77,6 +77,7 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { newArgs := append(args, execArgs...) ep.SetArgs(newArgs) + // TODO: Fix the race condition when executing commands in parallel wg := sync.WaitGroup{} mu := sync.Mutex{} errs := multierror.Error{} @@ -88,7 +89,7 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { go func(packageChan <-chan string) { defer wg.Done() for packageName := range packageChan { - if err := executeCommand(ep, args, root, packageName); err != nil { + if err := executeCommand(ep, root, packageName); err != nil { mu.Lock() errs = append(errs, fmt.Errorf("executing command for package %s failed: %w", packageName, err)) mu.Unlock() @@ -111,7 +112,7 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { return nil } -func executeCommand(ep *cobra.Command, args []string, root string, packageName string) error { +func executeCommand(ep *cobra.Command, root string, packageName string) error { // Set change directory flag to the package directory ep.Flags().Set(cobraext.ChangeDirectoryFlagName, filepath.Join(root, "packages", packageName)) diff --git a/internal/filter/category.go b/internal/filter/category.go index b70e19a602..8231daab11 100644 --- a/internal/filter/category.go +++ b/internal/filter/category.go @@ -7,18 +7,11 @@ import ( ) type CategoryFlag struct { - name string - description string - shorthand string - defaultValue string + FilterFlag values map[string]struct{} } -func (f *CategoryFlag) Register(cmd *cobra.Command) { - cmd.Flags().StringP(f.name, f.shorthand, f.defaultValue, f.description) -} - func (f *CategoryFlag) Parse(cmd *cobra.Command) error { category, err := cmd.Flags().GetString(cobraext.FilterCategoriesFlagName) if err != nil { @@ -47,9 +40,11 @@ func (f *CategoryFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []pack func initCategoryFlag() *CategoryFlag { return &CategoryFlag{ - name: cobraext.FilterCategoriesFlagName, - description: cobraext.FilterCategoriesFlagDescription, - shorthand: "", - defaultValue: "", + FilterFlag: FilterFlag{ + name: cobraext.FilterCategoriesFlagName, + description: cobraext.FilterCategoriesFlagDescription, + shorthand: "", + defaultValue: "", + }, } } diff --git a/internal/filter/codeowner.go b/internal/filter/codeowner.go index 41d8f7a0d5..750d2e166d 100644 --- a/internal/filter/codeowner.go +++ b/internal/filter/codeowner.go @@ -10,18 +10,10 @@ import ( ) type CodeOwnerFlag struct { - name string - description string - shorthand string - defaultValue string - + FilterFlag values map[string]struct{} } -func (f *CodeOwnerFlag) Register(cmd *cobra.Command) { - cmd.Flags().StringP(f.name, f.shorthand, f.defaultValue, f.description) -} - func (f *CodeOwnerFlag) Parse(cmd *cobra.Command) error { codeOwners, err := cmd.Flags().GetString(cobraext.FilterCodeOwnerFlagName) if err != nil { @@ -65,9 +57,11 @@ func (f *CodeOwnerFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []pac func initCodeOwnerFlag() *CodeOwnerFlag { return &CodeOwnerFlag{ - name: cobraext.FilterCodeOwnerFlagName, - description: cobraext.FilterCodeOwnerFlagDescription, - shorthand: "", - defaultValue: "", + FilterFlag: FilterFlag{ + name: cobraext.FilterCodeOwnerFlagName, + description: cobraext.FilterCodeOwnerFlagDescription, + shorthand: "", + defaultValue: "", + }, } } diff --git a/internal/filter/input.go b/internal/filter/input.go index bc8e82fe75..16923203d7 100644 --- a/internal/filter/input.go +++ b/internal/filter/input.go @@ -7,19 +7,12 @@ import ( ) type InputFlag struct { - name string - description string - shorthand string - defaultValue string + FilterFlag // flag specific fields values map[string]struct{} } -func (f *InputFlag) Register(cmd *cobra.Command) { - cmd.Flags().StringP(f.name, f.shorthand, f.defaultValue, f.description) -} - func (f *InputFlag) Parse(cmd *cobra.Command) error { input, err := cmd.Flags().GetString(cobraext.FilterInputFlagName) if err != nil { @@ -54,9 +47,11 @@ func (f *InputFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []package func initInputFlag() *InputFlag { return &InputFlag{ - name: cobraext.FilterInputFlagName, - description: cobraext.FilterInputFlagDescription, - shorthand: "", - defaultValue: "", + FilterFlag: FilterFlag{ + name: cobraext.FilterInputFlagName, + description: cobraext.FilterInputFlagDescription, + shorthand: "", + defaultValue: "", + }, } } diff --git a/internal/filter/registry.go b/internal/filter/registry.go index 1194295d71..e409af7503 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -9,29 +9,20 @@ import ( "github.com/spf13/cobra" ) -type FilterImpl interface { - Register(cmd *cobra.Command) - - Parse(cmd *cobra.Command) error - Validate() error - Matches(pkg packages.PackageManifest) bool - ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) -} - -var registry = []FilterImpl{ +var registry = []IFilter{ initInputFlag(), initCodeOwnerFlag(), initCategoryFlag(), } func SetFilterFlags(cmd *cobra.Command) { - for _, filter := range registry { - filter.Register(cmd) + for _, filterFlag := range registry { + filterFlag.Register(cmd) } } type FilterRegistry struct { - filters []FilterImpl + filters []IFilter } func NewFilterRegistry() *FilterRegistry { diff --git a/internal/filter/type.go b/internal/filter/type.go new file mode 100644 index 0000000000..4f7ecf2acb --- /dev/null +++ b/internal/filter/type.go @@ -0,0 +1,51 @@ +package filter + +import ( + "github.com/elastic/elastic-package/internal/packages" + "github.com/spf13/cobra" +) + +type IFilterFlag interface { + Name() string + Description() string + Shorthand() string + DefaultValue() string + + Register(cmd *cobra.Command) +} + +type IFilter interface { + IFilterFlag + + Parse(cmd *cobra.Command) error + Validate() error + Matches(pkg packages.PackageManifest) bool + ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) +} + +type FilterFlag struct { + name string + description string + shorthand string + defaultValue string +} + +func (f *FilterFlag) Name() string { + return f.name +} + +func (f *FilterFlag) Description() string { + return f.description +} + +func (f *FilterFlag) Shorthand() string { + return f.shorthand +} + +func (f *FilterFlag) DefaultValue() string { + return f.defaultValue +} + +func (f *FilterFlag) Register(cmd *cobra.Command) { + cmd.Flags().StringP(f.Name(), f.Shorthand(), f.DefaultValue(), f.Description()) +} From d94e9b2ebae136c5313dfc65bd6768869f390679 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Mon, 20 Oct 2025 14:59:16 -0400 Subject: [PATCH 19/36] added spec version flag --- internal/cobraext/flags.go | 3 ++ internal/filter/registry.go | 5 ++- internal/filter/specversion.go | 67 ++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 internal/filter/specversion.go diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 62f0a8c29c..d735f7d2bc 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -151,6 +151,9 @@ const ( FilterPackagesFlagName = "packages" FilterPackagesFlagDescription = "packages to filter by (glob pattern/comma-separated values)" + FilterSpecVersionFlagName = "spec-version" + FilterSpecVersionFlagDescription = "Package spec version to filter by (semver)" + ForeachExecFlagName = "exec" ForeachExecFlagDescription = "command to execute for each package (required)" diff --git a/internal/filter/registry.go b/internal/filter/registry.go index e409af7503..72f8e0f66d 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -10,9 +10,10 @@ import ( ) var registry = []IFilter{ - initInputFlag(), - initCodeOwnerFlag(), initCategoryFlag(), + initCodeOwnerFlag(), + initInputFlag(), + initSpecVersionFlag(), } func SetFilterFlags(cmd *cobra.Command) { diff --git a/internal/filter/specversion.go b/internal/filter/specversion.go new file mode 100644 index 0000000000..c77de67a3f --- /dev/null +++ b/internal/filter/specversion.go @@ -0,0 +1,67 @@ +package filter + +import ( + "fmt" + + "github.com/Masterminds/semver/v3" + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" + "github.com/spf13/cobra" +) + +type SpecVersionFlag struct { + FilterFlag + + // package spec version constraint + constraints *semver.Constraints +} + +func (f *SpecVersionFlag) Parse(cmd *cobra.Command) error { + + formatVersion, err := cmd.Flags().GetString(cobraext.FilterSpecVersionFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterSpecVersionFlagName) + } + + constraint, err := semver.NewConstraint(formatVersion) + if err != nil { + return fmt.Errorf("invalid format version: %s: %w", formatVersion, err) + } + + f.constraints = constraint + + return nil +} + +func (f *SpecVersionFlag) Validate() error { + // no validation needed for this flag + // checks are done in Parse method + return nil +} + +func (f *SpecVersionFlag) Matches(pkg packages.PackageManifest) bool { + // assuming the package spec version in manifest.yml is a valid semver + pkgVersion, _ := semver.NewVersion(pkg.SpecVersion) + + return f.constraints.Check(pkgVersion) +} + +func (f *SpecVersionFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { + for _, pkg := range pkgs { + if f.Matches(pkg) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initSpecVersionFlag() *SpecVersionFlag { + return &SpecVersionFlag{ + FilterFlag: FilterFlag{ + name: cobraext.FilterSpecVersionFlagName, + description: cobraext.FilterSpecVersionFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} From 3cc08249f971619c883c94c41949dbe23314d606 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Tue, 21 Oct 2025 12:54:36 -0400 Subject: [PATCH 20/36] fix issue with unused filters applied --- internal/filter/category.go | 5 +++++ internal/filter/codeowner.go | 8 ++++---- internal/filter/input.go | 5 +++++ internal/filter/registry.go | 9 +++++++-- internal/filter/specversion.go | 13 +++++++------ internal/filter/type.go | 7 +++++++ 6 files changed, 35 insertions(+), 12 deletions(-) diff --git a/internal/filter/category.go b/internal/filter/category.go index 8231daab11..da20c2715c 100644 --- a/internal/filter/category.go +++ b/internal/filter/category.go @@ -17,7 +17,12 @@ func (f *CategoryFlag) Parse(cmd *cobra.Command) error { if err != nil { return cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) } + if category == "" { + return nil + } + f.values = splitAndTrim(category, ",") + f.isApplied = true return nil } diff --git a/internal/filter/codeowner.go b/internal/filter/codeowner.go index 750d2e166d..ea0b5abac9 100644 --- a/internal/filter/codeowner.go +++ b/internal/filter/codeowner.go @@ -19,12 +19,12 @@ func (f *CodeOwnerFlag) Parse(cmd *cobra.Command) error { if err != nil { return cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) } - f.values = splitAndTrim(codeOwners, ",") - - if err := f.Validate(); err != nil { - return err + if codeOwners == "" { + return nil } + f.values = splitAndTrim(codeOwners, ",") + f.isApplied = true return nil } diff --git a/internal/filter/input.go b/internal/filter/input.go index 16923203d7..2955f56491 100644 --- a/internal/filter/input.go +++ b/internal/filter/input.go @@ -18,7 +18,12 @@ func (f *InputFlag) Parse(cmd *cobra.Command) error { if err != nil { return cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) } + if input == "" { + return nil + } + f.values = splitAndTrim(input, ",") + f.isApplied = true return nil } diff --git a/internal/filter/registry.go b/internal/filter/registry.go index 72f8e0f66d..fbba20467a 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -13,6 +13,7 @@ var registry = []IFilter{ initCategoryFlag(), initCodeOwnerFlag(), initInputFlag(), + initKibanaVersionFlag(), initSpecVersionFlag(), } @@ -28,16 +29,20 @@ type FilterRegistry struct { func NewFilterRegistry() *FilterRegistry { return &FilterRegistry{ - filters: registry, + filters: []IFilter{}, } } func (r *FilterRegistry) Parse(cmd *cobra.Command) error { errs := multierror.Error{} - for _, filter := range r.filters { + for _, filter := range registry { if err := filter.Parse(cmd); err != nil { errs = append(errs, err) } + + if filter.IsApplied() { + r.filters = append(r.filters, filter) + } } if errs.Error() != "" { diff --git a/internal/filter/specversion.go b/internal/filter/specversion.go index c77de67a3f..6fa5361e9e 100644 --- a/internal/filter/specversion.go +++ b/internal/filter/specversion.go @@ -17,19 +17,20 @@ type SpecVersionFlag struct { } func (f *SpecVersionFlag) Parse(cmd *cobra.Command) error { - - formatVersion, err := cmd.Flags().GetString(cobraext.FilterSpecVersionFlagName) + specVersion, err := cmd.Flags().GetString(cobraext.FilterSpecVersionFlagName) if err != nil { return cobraext.FlagParsingError(err, cobraext.FilterSpecVersionFlagName) } + if specVersion == "" { + return nil + } - constraint, err := semver.NewConstraint(formatVersion) + f.constraints, err = semver.NewConstraint(specVersion) if err != nil { - return fmt.Errorf("invalid format version: %s: %w", formatVersion, err) + return fmt.Errorf("invalid spec version: %s: %w", specVersion, err) } - f.constraints = constraint - + f.isApplied = true return nil } diff --git a/internal/filter/type.go b/internal/filter/type.go index 4f7ecf2acb..4758fce845 100644 --- a/internal/filter/type.go +++ b/internal/filter/type.go @@ -12,6 +12,7 @@ type IFilterFlag interface { DefaultValue() string Register(cmd *cobra.Command) + IsApplied() bool } type IFilter interface { @@ -28,6 +29,8 @@ type FilterFlag struct { description string shorthand string defaultValue string + + isApplied bool } func (f *FilterFlag) Name() string { @@ -49,3 +52,7 @@ func (f *FilterFlag) DefaultValue() string { func (f *FilterFlag) Register(cmd *cobra.Command) { cmd.Flags().StringP(f.Name(), f.Shorthand(), f.DefaultValue(), f.Description()) } + +func (f *FilterFlag) IsApplied() bool { + return f.isApplied +} From c36efd72b87ba2d7eb4fac5d6da4c1ec84636c5b Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Tue, 21 Oct 2025 15:26:52 -0400 Subject: [PATCH 21/36] run command in parallel --- cmd/foreach.go | 47 +++++++++++++++++++++++++------------ internal/filter/registry.go | 1 - 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/cmd/foreach.go b/cmd/foreach.go index 5f2e404e71..f25a871628 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -2,12 +2,16 @@ package cmd import ( "fmt" + "io" + "os" + "os/exec" "path/filepath" "strings" "sync" "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/filter" + "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" "github.com/spf13/cobra" @@ -69,30 +73,29 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { } fmt.Printf("Found %d matching package(s)\n", len(filtered)) - // Get elastic-package command - ep := cmd.Parent() - - // Split the exec command string into arguments and append the command arguments - execArgs := strings.Split(exec, " ") - newArgs := append(args, execArgs...) - ep.SetArgs(newArgs) - // TODO: Fix the race condition when executing commands in parallel wg := sync.WaitGroup{} mu := sync.Mutex{} errs := multierror.Error{} + successes := 0 packageChan := make(chan string, poolSize) + execArgs := strings.Split(exec, " ") for range poolSize { wg.Add(1) go func(packageChan <-chan string) { defer wg.Done() for packageName := range packageChan { - if err := executeCommand(ep, root, packageName); err != nil { + path := filepath.Join(root, "packages", packageName) + if err := executeCommand(execArgs, path); err != nil { mu.Lock() errs = append(errs, fmt.Errorf("executing command for package %s failed: %w", packageName, err)) mu.Unlock() + } else { + mu.Lock() + successes++ + mu.Unlock() } } }(packageChan) @@ -105,6 +108,9 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { wg.Wait() + logger.Infof("Successfully executed command for %d packages\n", successes) + logger.Infof("Errors occurred while executing command for %d packages\n", len(errs)) + if errs.Error() != "" { return fmt.Errorf("errors occurred while executing command for packages: \n%s", errs.Error()) } @@ -112,13 +118,24 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { return nil } -func executeCommand(ep *cobra.Command, root string, packageName string) error { - // Set change directory flag to the package directory - ep.Flags().Set(cobraext.ChangeDirectoryFlagName, filepath.Join(root, "packages", packageName)) +func executeCommand(args []string, path string) error { + // Look up the elastic-package binary in PATH + execPath, err := exec.LookPath("elastic-package") + if err != nil { + return fmt.Errorf("elastic-package binary not found in PATH: %w", err) + } + + cmd := exec.Cmd{ + Path: execPath, + Args: append([]string{"elastic-package"}, args...), + Dir: path, + Stdout: io.Discard, + Stderr: os.Stderr, + Env: os.Environ(), + } - // Execute command - if err := ep.Execute(); err != nil { - return fmt.Errorf("executing command for package %s failed: %w", packageName, err) + if err := cmd.Run(); err != nil { + return fmt.Errorf("executing command for package %s failed: %w", path, err) } return nil diff --git a/internal/filter/registry.go b/internal/filter/registry.go index fbba20467a..c0fcbc3738 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -13,7 +13,6 @@ var registry = []IFilter{ initCategoryFlag(), initCodeOwnerFlag(), initInputFlag(), - initKibanaVersionFlag(), initSpecVersionFlag(), } From 0649c73fc308eafae18867153e2f7a48a7a09040 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Wed, 22 Oct 2025 14:14:47 -0400 Subject: [PATCH 22/36] remove --exec to leverage -- to pass sub commands without string --- cmd/foreach.go | 30 +++++++++--------------------- internal/cobraext/flags.go | 3 --- internal/filter/registry.go | 2 +- 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/cmd/foreach.go b/cmd/foreach.go index f25a871628..c8cfb6d01f 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -6,7 +6,6 @@ import ( "os" "os/exec" "path/filepath" - "strings" "sync" "github.com/elastic/elastic-package/internal/cobraext" @@ -27,13 +26,13 @@ then executes the specified subcommand for each matched package.` func setupForeachCommand() *cobraext.Command { cmd := &cobra.Command{ - Use: "foreach --exec ''", + Use: "foreach [flags] -- ", Short: "Execute a command for filtered packages", Long: foreachLongDescription, Example: ` # Run system tests for packages with specific inputs - elastic-package foreach --exec 'test system -g' --input tcp,udp`, + elastic-package foreach --input tcp,udp --parallel 10 -- test system -g`, RunE: foreachCommandAction, - Args: cobra.NoArgs, + Args: cobra.MinimumNArgs(1), } // Add filter flags @@ -42,19 +41,10 @@ func setupForeachCommand() *cobraext.Command { // Add pool size flag cmd.Flags().IntP(cobraext.ForeachPoolSizeFlagName, "", 1, cobraext.ForeachPoolSizeFlagDescription) - // Add exec flag and mark it as required - cmd.Flags().StringP(cobraext.ForeachExecFlagName, "", "", cobraext.ForeachExecFlagDescription) - cmd.MarkFlagRequired(cobraext.ForeachExecFlagName) - return cobraext.NewCommand(cmd, cobraext.ContextPackage) } func foreachCommandAction(cmd *cobra.Command, args []string) error { - exec, err := cmd.Flags().GetString(cobraext.ForeachExecFlagName) - if err != nil { - return fmt.Errorf("getting exec failed: %w", err) - } - poolSize, err := cmd.Flags().GetInt(cobraext.ForeachPoolSizeFlagName) if err != nil { return fmt.Errorf("getting pool size failed: %w", err) @@ -71,7 +61,6 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("filtering packages failed: %w", err) } - fmt.Printf("Found %d matching package(s)\n", len(filtered)) // TODO: Fix the race condition when executing commands in parallel wg := sync.WaitGroup{} @@ -81,22 +70,21 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { packageChan := make(chan string, poolSize) - execArgs := strings.Split(exec, " ") for range poolSize { wg.Add(1) go func(packageChan <-chan string) { defer wg.Done() for packageName := range packageChan { path := filepath.Join(root, "packages", packageName) - if err := executeCommand(execArgs, path); err != nil { - mu.Lock() + err := executeCommand(args, path) + + mu.Lock() + if err != nil { errs = append(errs, fmt.Errorf("executing command for package %s failed: %w", packageName, err)) - mu.Unlock() } else { - mu.Lock() successes++ - mu.Unlock() } + mu.Unlock() } }(packageChan) } @@ -127,7 +115,7 @@ func executeCommand(args []string, path string) error { cmd := exec.Cmd{ Path: execPath, - Args: append([]string{"elastic-package"}, args...), + Args: append([]string{execPath}, args...), Dir: path, Stdout: io.Discard, Stderr: os.Stderr, diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index d735f7d2bc..cb971d0535 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -154,9 +154,6 @@ const ( FilterSpecVersionFlagName = "spec-version" FilterSpecVersionFlagDescription = "Package spec version to filter by (semver)" - ForeachExecFlagName = "exec" - ForeachExecFlagDescription = "command to execute for each package (required)" - ForeachPoolSizeFlagName = "parallel" ForeachPoolSizeFlagDescription = "number of packages to execute in parallel (defaults to serial execution)" diff --git a/internal/filter/registry.go b/internal/filter/registry.go index c0fcbc3738..dcbe0d40bf 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -79,6 +79,6 @@ func (r *FilterRegistry) Execute() (filtered []packages.PackageManifest, err err } } - logger.Infof("Filtered %d packages\n", len(filtered)) + logger.Infof("Found %d matching package(s)\n", len(filtered)) return filtered, nil } From 5e3b8b0e5694ea9bac313b427c8b3d78405102d6 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Wed, 22 Oct 2025 14:45:45 -0400 Subject: [PATCH 23/36] add subcommand allowlist --- cmd/foreach.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cmd/foreach.go b/cmd/foreach.go index c8cfb6d01f..8eb841f30c 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -6,6 +6,8 @@ import ( "os" "os/exec" "path/filepath" + "slices" + "strings" "sync" "github.com/elastic/elastic-package/internal/cobraext" @@ -50,6 +52,10 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("getting pool size failed: %w", err) } + if err := validateSubCommand(args[0]); err != nil { + return fmt.Errorf("validating sub command failed: %w", err) + } + // Find integration root root, err := packages.MustFindIntegrationRoot() if err != nil { @@ -128,3 +134,22 @@ func executeCommand(args []string, path string) error { return nil } + +func validateSubCommand(subCommand string) error { + allowedSubCommands := []string{ + "build", + "check", + "clean", + "format", + "install", + "lint", + "test", + "uninstall", + } + + if !slices.Contains(allowedSubCommands, subCommand) { + return fmt.Errorf("invalid subcommand: %s. Allowed subcommands are: %s", subCommand, strings.Join(allowedSubCommands, ", ")) + } + + return nil +} From df2c85b91e3aedefcf7bdd3e1fe2b00e97635f13 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Wed, 22 Oct 2025 15:55:28 -0400 Subject: [PATCH 24/36] create map of dir_name and manifest to allow different package_name in manifest and dir_name in integrations repo. --- cmd/filter.go | 35 +++++++++++++++++++++++++--------- cmd/foreach.go | 4 ++-- internal/cobraext/flags.go | 4 ++++ internal/filter/category.go | 15 ++++++++------- internal/filter/codeowner.go | 13 +++++++------ internal/filter/input.go | 14 ++++++++------ internal/filter/registry.go | 10 +++++----- internal/filter/specversion.go | 12 +++++++----- internal/filter/type.go | 5 +++-- internal/packages/packages.go | 7 ++++--- 10 files changed, 74 insertions(+), 45 deletions(-) diff --git a/cmd/filter.go b/cmd/filter.go index 76a2f42aae..98b32451e0 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "os" + "slices" "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/filter" @@ -30,6 +31,9 @@ func setupFilterCommand() *cobraext.Command { // add filter flags to the command (input, code owner, kibana version, categories) filter.SetFilterFlags(cmd) + // add the output package name flag to the command + cmd.Flags().BoolP(cobraext.FilterOutputPackageNameFlagName, cobraext.FilterOutputPackageNameFlagShorthand, false, cobraext.FilterOutputPackageNameFlagDescription) + return cobraext.NewCommand(cmd, cobraext.ContextPackage) } @@ -39,35 +43,41 @@ func filterCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("filtering packages failed: %w", err) } - if err = printPkgList(filtered, os.Stdout); err != nil { + printPackageName, err := cmd.Flags().GetBool(cobraext.FilterOutputPackageNameFlagName) + if err != nil { + return fmt.Errorf("getting output package name flag failed: %w", err) + } + + if err = printPkgList(filtered, printPackageName, os.Stdout); err != nil { return fmt.Errorf("printing JSON failed: %w", err) } return nil } -func filterPackage(cmd *cobra.Command) ([]packages.PackageManifest, error) { - var filtered []packages.PackageManifest +func filterPackage(cmd *cobra.Command) (map[string]packages.PackageManifest, error) { + var filtered map[string]packages.PackageManifest var err error filters := filter.NewFilterRegistry() if err = filters.Parse(cmd); err != nil { - return nil, fmt.Errorf("getting filter options failed: %w", err) + return nil, fmt.Errorf("parsing filter options failed: %w", err) } if err = filters.Validate(); err != nil { return nil, fmt.Errorf("validating filter options failed: %w", err) } - if filtered, err = filters.Execute(); err != nil { - return nil, fmt.Errorf("filtering packages failed: %w", err) + filtered, errors := filters.Execute() + if errors != nil { + return nil, fmt.Errorf("filtering packages failed: %s", errors.Error()) } return filtered, nil } -func printPkgList(pkgs []packages.PackageManifest, w io.Writer) error { +func printPkgList(pkgs map[string]packages.PackageManifest, printPackageName bool, w io.Writer) error { enc := json.NewEncoder(w) enc.SetEscapeHTML(false) if len(pkgs) == 0 { @@ -75,9 +85,16 @@ func printPkgList(pkgs []packages.PackageManifest, w io.Writer) error { } names := make([]string, 0, len(pkgs)) - for _, pkg := range pkgs { - names = append(names, pkg.Name) + if printPackageName { + for _, pkgManifest := range pkgs { + names = append(names, pkgManifest.Name) + } + } else { + for pkgDirName := range pkgs { + names = append(names, pkgDirName) + } } + slices.Sort(names) return enc.Encode(names) } diff --git a/cmd/foreach.go b/cmd/foreach.go index 8eb841f30c..ff2a49a254 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -95,8 +95,8 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { }(packageChan) } - for _, pkg := range filtered { - packageChan <- pkg.Name + for pkgName := range filtered { + packageChan <- pkgName } close(packageChan) diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index cb971d0535..96166447c5 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -148,6 +148,10 @@ const ( FilterKibanaVersionFlagName = "kibana-version" FilterKibanaVersionFlagDescription = "kibana version to filter by (semver)" + FilterOutputPackageNameFlagName = "output-package-name" + FilterOutputPackageNameFlagShorthand = "p" + FilterOutputPackageNameFlagDescription = "print the package name instead of the directory name in the output" + FilterPackagesFlagName = "packages" FilterPackagesFlagDescription = "packages to filter by (glob pattern/comma-separated values)" diff --git a/internal/filter/category.go b/internal/filter/category.go index da20c2715c..302766659d 100644 --- a/internal/filter/category.go +++ b/internal/filter/category.go @@ -30,17 +30,18 @@ func (f *CategoryFlag) Validate() error { return nil } -func (f *CategoryFlag) Matches(pkg packages.PackageManifest) bool { - return hasAnyMatch(f.values, pkg.Categories) +func (f *CategoryFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { + return hasAnyMatch(f.values, pkgManifest.Categories) } -func (f *CategoryFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { - for _, pkg := range pkgs { - if f.Matches(pkg) { - filtered = append(filtered, pkg) +func (f *CategoryFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) { + filtered := make(map[string]packages.PackageManifest, len(pkgs)) + for pkgDirName, pkgManifest := range pkgs { + if f.Matches(pkgDirName, pkgManifest) { + filtered[pkgDirName] = pkgManifest } } - return filtered, err + return filtered, nil } func initCategoryFlag() *CategoryFlag { diff --git a/internal/filter/codeowner.go b/internal/filter/codeowner.go index ea0b5abac9..970b0e5936 100644 --- a/internal/filter/codeowner.go +++ b/internal/filter/codeowner.go @@ -42,14 +42,15 @@ func (f *CodeOwnerFlag) Validate() error { return nil } -func (f *CodeOwnerFlag) Matches(pkg packages.PackageManifest) bool { - return hasAnyMatch(f.values, []string{pkg.Owner.Github}) +func (f *CodeOwnerFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { + return hasAnyMatch(f.values, []string{pkgManifest.Owner.Github}) } -func (f *CodeOwnerFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { - for _, pkg := range pkgs { - if f.Matches(pkg) { - filtered = append(filtered, pkg) +func (f *CodeOwnerFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) { + filtered := make(map[string]packages.PackageManifest, len(pkgs)) + for pkgDirName, pkgManifest := range pkgs { + if f.Matches(pkgDirName, pkgManifest) { + filtered[pkgDirName] = pkgManifest } } return filtered, nil diff --git a/internal/filter/input.go b/internal/filter/input.go index 2955f56491..2b6f1751e1 100644 --- a/internal/filter/input.go +++ b/internal/filter/input.go @@ -31,9 +31,9 @@ func (f *InputFlag) Validate() error { return nil } -func (f *InputFlag) Matches(pkg packages.PackageManifest) bool { +func (f *InputFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { if f.values != nil { - inputs := extractInputs(pkg) + inputs := extractInputs(pkgManifest) if !hasAnyMatch(f.values, inputs) { return false } @@ -41,10 +41,12 @@ func (f *InputFlag) Matches(pkg packages.PackageManifest) bool { return true } -func (f *InputFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { - for _, pkg := range pkgs { - if f.Matches(pkg) { - filtered = append(filtered, pkg) +func (f *InputFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) { + filtered := make(map[string]packages.PackageManifest, len(pkgs)) + + for pkgName, pkgManifest := range pkgs { + if f.Matches(pkgName, pkgManifest) { + filtered[pkgName] = pkgManifest } } return filtered, nil diff --git a/internal/filter/registry.go b/internal/filter/registry.go index dcbe0d40bf..c68b806690 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -60,25 +60,25 @@ func (r *FilterRegistry) Validate() error { return nil } -func (r *FilterRegistry) Execute() (filtered []packages.PackageManifest, err error) { +func (r *FilterRegistry) Execute() (filtered map[string]packages.PackageManifest, errors multierror.Error) { root, err := packages.MustFindIntegrationRoot() if err != nil { - return nil, err + return nil, multierror.Error{err} } pkgs, err := packages.ReadAllPackageManifests(root) if err != nil { - return nil, err + return nil, multierror.Error{err} } filtered = pkgs for _, filter := range r.filters { filtered, err = filter.ApplyTo(filtered) if err != nil || len(filtered) == 0 { - break + errors = append(errors, err) } } logger.Infof("Found %d matching package(s)\n", len(filtered)) - return filtered, nil + return filtered, errors } diff --git a/internal/filter/specversion.go b/internal/filter/specversion.go index 6fa5361e9e..235c4f5c47 100644 --- a/internal/filter/specversion.go +++ b/internal/filter/specversion.go @@ -40,17 +40,19 @@ func (f *SpecVersionFlag) Validate() error { return nil } -func (f *SpecVersionFlag) Matches(pkg packages.PackageManifest) bool { +func (f *SpecVersionFlag) Matches(pkgName string, pkg packages.PackageManifest) bool { // assuming the package spec version in manifest.yml is a valid semver pkgVersion, _ := semver.NewVersion(pkg.SpecVersion) return f.constraints.Check(pkgVersion) } -func (f *SpecVersionFlag) ApplyTo(pkgs []packages.PackageManifest) (filtered []packages.PackageManifest, err error) { - for _, pkg := range pkgs { - if f.Matches(pkg) { - filtered = append(filtered, pkg) +func (f *SpecVersionFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (filtered map[string]packages.PackageManifest, err error) { + filtered = make(map[string]packages.PackageManifest, len(pkgs)) + + for pkgName, pkg := range pkgs { + if f.Matches(pkgName, pkg) { + filtered[pkgName] = pkg } } return filtered, nil diff --git a/internal/filter/type.go b/internal/filter/type.go index 4758fce845..099bf5fec9 100644 --- a/internal/filter/type.go +++ b/internal/filter/type.go @@ -20,8 +20,9 @@ type IFilter interface { Parse(cmd *cobra.Command) error Validate() error - Matches(pkg packages.PackageManifest) bool - ApplyTo(pkgs []packages.PackageManifest) ([]packages.PackageManifest, error) + ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) + // pkgDirName is the directory name of the package in package root + Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool } type FilterFlag struct { diff --git a/internal/packages/packages.go b/internal/packages/packages.go index d5f4eecd93..d31fef51c3 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -471,19 +471,20 @@ func ReadPackageManifest(path string) (*PackageManifest, error) { } // ReadAllPackageManifests reads all the package manifests in the given root directory. -func ReadAllPackageManifests(root string) ([]PackageManifest, error) { +func ReadAllPackageManifests(root string) (map[string]PackageManifest, error) { files, err := filepath.Glob(filepath.Join(root, "packages", "*", PackageManifestFile)) if err != nil { return nil, fmt.Errorf("failed matching files with package manifests: %w", err) } - var packages []PackageManifest + packages := make(map[string]PackageManifest, len(files)) for _, file := range files { manifest, err := ReadPackageManifest(file) if err != nil { return nil, fmt.Errorf("failed to read package manifest: %w", err) } - packages = append(packages, *manifest) + + packages[filepath.Base(filepath.Dir(file))] = *manifest } return packages, nil From b495ced53ce20a64a8fe7a920dad12e356a6bbfc Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Wed, 22 Oct 2025 16:15:49 -0400 Subject: [PATCH 25/36] add flag suffix to go files --- internal/filter/{category.go => category_flag.go} | 0 internal/filter/{codeowner.go => codeowner_flag.go} | 0 internal/filter/{input.go => input_flag.go} | 0 internal/filter/{specversion.go => specversion_flag.go} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename internal/filter/{category.go => category_flag.go} (100%) rename internal/filter/{codeowner.go => codeowner_flag.go} (100%) rename internal/filter/{input.go => input_flag.go} (100%) rename internal/filter/{specversion.go => specversion_flag.go} (100%) diff --git a/internal/filter/category.go b/internal/filter/category_flag.go similarity index 100% rename from internal/filter/category.go rename to internal/filter/category_flag.go diff --git a/internal/filter/codeowner.go b/internal/filter/codeowner_flag.go similarity index 100% rename from internal/filter/codeowner.go rename to internal/filter/codeowner_flag.go diff --git a/internal/filter/input.go b/internal/filter/input_flag.go similarity index 100% rename from internal/filter/input.go rename to internal/filter/input_flag.go diff --git a/internal/filter/specversion.go b/internal/filter/specversion_flag.go similarity index 100% rename from internal/filter/specversion.go rename to internal/filter/specversion_flag.go From 812530c486ca41238a95b9171db859e69dac305c Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Wed, 22 Oct 2025 16:16:03 -0400 Subject: [PATCH 26/36] add integration type flag --- internal/cobraext/flags.go | 3 ++ internal/filter/integrationtype_flag.go | 62 +++++++++++++++++++++++++ internal/filter/registry.go | 1 + 3 files changed, 66 insertions(+) create mode 100644 internal/filter/integrationtype_flag.go diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 96166447c5..ceaa6322f4 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -142,6 +142,9 @@ const ( FilterCodeOwnerFlagName = "code-owner" FilterCodeOwnerFlagDescription = "code owners to filter by (comma-separated values)" + FilterIntegrationTypeFlagName = "integration-type" + FilterIntegrationTypeFlagDescription = "integration types to filter by (comma-separated values)" + FilterInputFlagName = "input" FilterInputFlagDescription = "name of the inputs to filter by (comma-separated values)" diff --git a/internal/filter/integrationtype_flag.go b/internal/filter/integrationtype_flag.go new file mode 100644 index 0000000000..8109e39068 --- /dev/null +++ b/internal/filter/integrationtype_flag.go @@ -0,0 +1,62 @@ +package filter + +import ( + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" + "github.com/spf13/cobra" +) + +type IntegrationTypeFlag struct { + FilterFlag + + // flag specific fields + values map[string]struct{} +} + +func (f *IntegrationTypeFlag) Parse(cmd *cobra.Command) error { + integrationTypes, err := cmd.Flags().GetString(cobraext.FilterIntegrationTypeFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterIntegrationTypeFlagName) + } + if integrationTypes == "" { + return nil + } + f.values = splitAndTrim(integrationTypes, ",") + f.isApplied = true + return nil +} + +func (f *IntegrationTypeFlag) Validate() error { + return nil +} + +func (f *IntegrationTypeFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { + if f.values != nil { + if !hasAnyMatch(f.values, []string{pkgManifest.Type}) { + return false + } + } + return true +} + +func (f *IntegrationTypeFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) { + filtered := make(map[string]packages.PackageManifest, len(pkgs)) + + for pkgName, pkgManifest := range pkgs { + if f.Matches(pkgName, pkgManifest) { + filtered[pkgName] = pkgManifest + } + } + return filtered, nil +} + +func initIntegrationTypeFlag() *IntegrationTypeFlag { + return &IntegrationTypeFlag{ + FilterFlag: FilterFlag{ + name: cobraext.FilterIntegrationTypeFlagName, + description: cobraext.FilterIntegrationTypeFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/registry.go b/internal/filter/registry.go index c68b806690..4a88ac71c7 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -13,6 +13,7 @@ var registry = []IFilter{ initCategoryFlag(), initCodeOwnerFlag(), initInputFlag(), + initIntegrationTypeFlag(), initSpecVersionFlag(), } From e3e3c9a1bafe508821d155475d9fccc093eab5e1 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Wed, 22 Oct 2025 16:42:00 -0400 Subject: [PATCH 27/36] add package name flag --- internal/cobraext/flags.go | 2 +- internal/filter/packagename_flag.go | 69 +++++++++++++++++++++++++++++ internal/filter/registry.go | 1 + 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 internal/filter/packagename_flag.go diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index ceaa6322f4..c2c8b31b6e 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -156,7 +156,7 @@ const ( FilterOutputPackageNameFlagDescription = "print the package name instead of the directory name in the output" FilterPackagesFlagName = "packages" - FilterPackagesFlagDescription = "packages to filter by (glob pattern/comma-separated values)" + FilterPackagesFlagDescription = "regex pattern to filter packages by" FilterSpecVersionFlagName = "spec-version" FilterSpecVersionFlagDescription = "Package spec version to filter by (semver)" diff --git a/internal/filter/packagename_flag.go b/internal/filter/packagename_flag.go new file mode 100644 index 0000000000..51428944ec --- /dev/null +++ b/internal/filter/packagename_flag.go @@ -0,0 +1,69 @@ +package filter + +import ( + "fmt" + "regexp" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" + "github.com/spf13/cobra" +) + +type PackageNameFlag struct { + FilterFlag + + patterns []*regexp.Regexp +} + +func (f *PackageNameFlag) Parse(cmd *cobra.Command) error { + packageNamePatterns, err := cmd.Flags().GetString(cobraext.FilterPackagesFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterPackagesFlagName) + } + + patternStrings := splitAndTrim(packageNamePatterns, ",") + for patternString := range patternStrings { + regex, err := regexp.Compile(patternString) + if err != nil { + return fmt.Errorf("invalid package name pattern: %s: %w", patternString, err) + } + f.patterns = append(f.patterns, regex) + } + + f.isApplied = true + return nil +} + +func (f *PackageNameFlag) Validate() error { + return nil +} + +func (f *PackageNameFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { + for _, pattern := range f.patterns { + if pattern.MatchString(pkgDirName) { + return true + } + } + return false +} + +func (f *PackageNameFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) { + filtered := make(map[string]packages.PackageManifest, len(pkgs)) + for pkgDirName, pkgManifest := range pkgs { + if f.Matches(pkgDirName, pkgManifest) { + filtered[pkgDirName] = pkgManifest + } + } + return filtered, nil +} + +func initPackageNameFlag() *PackageNameFlag { + return &PackageNameFlag{ + FilterFlag: FilterFlag{ + name: cobraext.FilterPackagesFlagName, + description: cobraext.FilterPackagesFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/registry.go b/internal/filter/registry.go index 4a88ac71c7..f93687e910 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -14,6 +14,7 @@ var registry = []IFilter{ initCodeOwnerFlag(), initInputFlag(), initIntegrationTypeFlag(), + initPackageNameFlag(), initSpecVersionFlag(), } From 54fce2b1abdf1e91edd3e6a1fa7155426ae7417f Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 23 Oct 2025 12:19:16 -0400 Subject: [PATCH 28/36] minor refactors --- cmd/filter.go | 9 +++------ cmd/foreach.go | 7 +++++-- internal/filter/category_flag.go | 4 ++-- internal/filter/codeowner_flag.go | 4 ++-- internal/filter/input_flag.go | 4 ++-- internal/filter/integrationtype_flag.go | 4 ++-- internal/filter/packagename_flag.go | 9 ++++++--- internal/filter/registry.go | 15 ++++++++++---- internal/filter/specversion_flag.go | 11 +++++----- internal/filter/type.go | 27 +++++++++++++++---------- 10 files changed, 55 insertions(+), 39 deletions(-) diff --git a/cmd/filter.go b/cmd/filter.go index 98b32451e0..9f1e83e7f4 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -17,7 +17,7 @@ import ( "github.com/spf13/cobra" ) -const filterLongDescription = `This command would give you a list of all the packages based on the given query` +const filterLongDescription = `This command gives you a list of all packages based on the given query` func setupFilterCommand() *cobraext.Command { cmd := &cobra.Command{ @@ -56,16 +56,13 @@ func filterCommandAction(cmd *cobra.Command, args []string) error { } func filterPackage(cmd *cobra.Command) (map[string]packages.PackageManifest, error) { - var filtered map[string]packages.PackageManifest - var err error - filters := filter.NewFilterRegistry() - if err = filters.Parse(cmd); err != nil { + if err := filters.Parse(cmd); err != nil { return nil, fmt.Errorf("parsing filter options failed: %w", err) } - if err = filters.Validate(); err != nil { + if err := filters.Validate(); err != nil { return nil, fmt.Errorf("validating filter options failed: %w", err) } diff --git a/cmd/foreach.go b/cmd/foreach.go index ff2a49a254..c9c01f98eb 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package cmd import ( @@ -68,7 +72,6 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("filtering packages failed: %w", err) } - // TODO: Fix the race condition when executing commands in parallel wg := sync.WaitGroup{} mu := sync.Mutex{} errs := multierror.Error{} @@ -119,7 +122,7 @@ func executeCommand(args []string, path string) error { return fmt.Errorf("elastic-package binary not found in PATH: %w", err) } - cmd := exec.Cmd{ + cmd := &exec.Cmd{ Path: execPath, Args: append([]string{execPath}, args...), Dir: path, diff --git a/internal/filter/category_flag.go b/internal/filter/category_flag.go index 302766659d..8361e99a3e 100644 --- a/internal/filter/category_flag.go +++ b/internal/filter/category_flag.go @@ -7,7 +7,7 @@ import ( ) type CategoryFlag struct { - FilterFlag + FilterFlagBase values map[string]struct{} } @@ -46,7 +46,7 @@ func (f *CategoryFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[st func initCategoryFlag() *CategoryFlag { return &CategoryFlag{ - FilterFlag: FilterFlag{ + FilterFlagBase: FilterFlagBase{ name: cobraext.FilterCategoriesFlagName, description: cobraext.FilterCategoriesFlagDescription, shorthand: "", diff --git a/internal/filter/codeowner_flag.go b/internal/filter/codeowner_flag.go index 970b0e5936..417c153b86 100644 --- a/internal/filter/codeowner_flag.go +++ b/internal/filter/codeowner_flag.go @@ -10,7 +10,7 @@ import ( ) type CodeOwnerFlag struct { - FilterFlag + FilterFlagBase values map[string]struct{} } @@ -58,7 +58,7 @@ func (f *CodeOwnerFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[s func initCodeOwnerFlag() *CodeOwnerFlag { return &CodeOwnerFlag{ - FilterFlag: FilterFlag{ + FilterFlagBase: FilterFlagBase{ name: cobraext.FilterCodeOwnerFlagName, description: cobraext.FilterCodeOwnerFlagDescription, shorthand: "", diff --git a/internal/filter/input_flag.go b/internal/filter/input_flag.go index 2b6f1751e1..57726fdf99 100644 --- a/internal/filter/input_flag.go +++ b/internal/filter/input_flag.go @@ -7,7 +7,7 @@ import ( ) type InputFlag struct { - FilterFlag + FilterFlagBase // flag specific fields values map[string]struct{} @@ -54,7 +54,7 @@ func (f *InputFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[strin func initInputFlag() *InputFlag { return &InputFlag{ - FilterFlag: FilterFlag{ + FilterFlagBase: FilterFlagBase{ name: cobraext.FilterInputFlagName, description: cobraext.FilterInputFlagDescription, shorthand: "", diff --git a/internal/filter/integrationtype_flag.go b/internal/filter/integrationtype_flag.go index 8109e39068..819617a3ec 100644 --- a/internal/filter/integrationtype_flag.go +++ b/internal/filter/integrationtype_flag.go @@ -7,7 +7,7 @@ import ( ) type IntegrationTypeFlag struct { - FilterFlag + FilterFlagBase // flag specific fields values map[string]struct{} @@ -52,7 +52,7 @@ func (f *IntegrationTypeFlag) ApplyTo(pkgs map[string]packages.PackageManifest) func initIntegrationTypeFlag() *IntegrationTypeFlag { return &IntegrationTypeFlag{ - FilterFlag: FilterFlag{ + FilterFlagBase: FilterFlagBase{ name: cobraext.FilterIntegrationTypeFlagName, description: cobraext.FilterIntegrationTypeFlagDescription, shorthand: "", diff --git a/internal/filter/packagename_flag.go b/internal/filter/packagename_flag.go index 51428944ec..1f9f4e89c6 100644 --- a/internal/filter/packagename_flag.go +++ b/internal/filter/packagename_flag.go @@ -10,7 +10,7 @@ import ( ) type PackageNameFlag struct { - FilterFlag + FilterFlagBase patterns []*regexp.Regexp } @@ -30,7 +30,10 @@ func (f *PackageNameFlag) Parse(cmd *cobra.Command) error { f.patterns = append(f.patterns, regex) } - f.isApplied = true + if len(f.patterns) > 0 { + f.isApplied = true + } + return nil } @@ -59,7 +62,7 @@ func (f *PackageNameFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map func initPackageNameFlag() *PackageNameFlag { return &PackageNameFlag{ - FilterFlag: FilterFlag{ + FilterFlagBase: FilterFlagBase{ name: cobraext.FilterPackagesFlagName, description: cobraext.FilterPackagesFlagDescription, shorthand: "", diff --git a/internal/filter/registry.go b/internal/filter/registry.go index f93687e910..6dc5b6ad11 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -var registry = []IFilter{ +var registry = []Filter{ initCategoryFlag(), initCodeOwnerFlag(), initInputFlag(), @@ -18,19 +18,22 @@ var registry = []IFilter{ initSpecVersionFlag(), } +// SetFilterFlags registers all filter flags with the given command. func SetFilterFlags(cmd *cobra.Command) { for _, filterFlag := range registry { filterFlag.Register(cmd) } } +// FilterRegistry manages a collection of filters for package filtering. type FilterRegistry struct { - filters []IFilter + filters []Filter } +// NewFilterRegistry creates a new FilterRegistry instance. func NewFilterRegistry() *FilterRegistry { return &FilterRegistry{ - filters: []IFilter{}, + filters: []Filter{}, } } @@ -76,9 +79,13 @@ func (r *FilterRegistry) Execute() (filtered map[string]packages.PackageManifest filtered = pkgs for _, filter := range r.filters { filtered, err = filter.ApplyTo(filtered) - if err != nil || len(filtered) == 0 { + if err != nil { errors = append(errors, err) } + + if len(filtered) == 0 { + break + } } logger.Infof("Found %d matching package(s)\n", len(filtered)) diff --git a/internal/filter/specversion_flag.go b/internal/filter/specversion_flag.go index 235c4f5c47..deb9d134c4 100644 --- a/internal/filter/specversion_flag.go +++ b/internal/filter/specversion_flag.go @@ -10,7 +10,7 @@ import ( ) type SpecVersionFlag struct { - FilterFlag + FilterFlagBase // package spec version constraint constraints *semver.Constraints @@ -41,9 +41,10 @@ func (f *SpecVersionFlag) Validate() error { } func (f *SpecVersionFlag) Matches(pkgName string, pkg packages.PackageManifest) bool { - // assuming the package spec version in manifest.yml is a valid semver - pkgVersion, _ := semver.NewVersion(pkg.SpecVersion) - + pkgVersion, err := semver.NewVersion(pkg.SpecVersion) + if err != nil { + return false + } return f.constraints.Check(pkgVersion) } @@ -60,7 +61,7 @@ func (f *SpecVersionFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (fil func initSpecVersionFlag() *SpecVersionFlag { return &SpecVersionFlag{ - FilterFlag: FilterFlag{ + FilterFlagBase: FilterFlagBase{ name: cobraext.FilterSpecVersionFlagName, description: cobraext.FilterSpecVersionFlagDescription, shorthand: "", diff --git a/internal/filter/type.go b/internal/filter/type.go index 099bf5fec9..ef191478df 100644 --- a/internal/filter/type.go +++ b/internal/filter/type.go @@ -5,7 +5,8 @@ import ( "github.com/spf13/cobra" ) -type IFilterFlag interface { +// FilterFlag defines the basic interface for filter flags. +type FilterFlag interface { Name() string Description() string Shorthand() string @@ -15,17 +16,21 @@ type IFilterFlag interface { IsApplied() bool } -type IFilter interface { - IFilterFlag +// Filter extends FilterFlag with filtering capabilities. +// It defines the interface for filtering packages based on specific criteria. +type Filter interface { + FilterFlag Parse(cmd *cobra.Command) error Validate() error ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) - // pkgDirName is the directory name of the package in package root + // Matches checks if a package matches the filter criteria. + // pkgDirName is the directory name of the package in package root. Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool } -type FilterFlag struct { +// FilterFlagBase provides common functionality for filter flags. +type FilterFlagBase struct { name string description string shorthand string @@ -34,26 +39,26 @@ type FilterFlag struct { isApplied bool } -func (f *FilterFlag) Name() string { +func (f *FilterFlagBase) Name() string { return f.name } -func (f *FilterFlag) Description() string { +func (f *FilterFlagBase) Description() string { return f.description } -func (f *FilterFlag) Shorthand() string { +func (f *FilterFlagBase) Shorthand() string { return f.shorthand } -func (f *FilterFlag) DefaultValue() string { +func (f *FilterFlagBase) DefaultValue() string { return f.defaultValue } -func (f *FilterFlag) Register(cmd *cobra.Command) { +func (f *FilterFlagBase) Register(cmd *cobra.Command) { cmd.Flags().StringP(f.Name(), f.Shorthand(), f.DefaultValue(), f.Description()) } -func (f *FilterFlag) IsApplied() bool { +func (f *FilterFlagBase) IsApplied() bool { return f.isApplied } From 13f348ffb5029dcef5ef71ac3a5c2969563963d9 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Fri, 24 Oct 2025 13:31:44 -0400 Subject: [PATCH 29/36] use glob instead of regex. --- go.mod | 1 + go.sum | 2 ++ internal/filter/packagename_flag.go | 14 +++++++------- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index acf569a3cb..346fbe1102 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/elastic/package-spec/v3 v3.5.0 github.com/fatih/color v1.18.0 github.com/go-viper/mapstructure/v2 v2.4.0 + github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.7.0 github.com/google/go-github/v32 v32.1.0 github.com/google/go-querystring v1.1.0 diff --git a/go.sum b/go.sum index b34f5a5d5c..c06a0c3656 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= diff --git a/internal/filter/packagename_flag.go b/internal/filter/packagename_flag.go index 1f9f4e89c6..478f27674a 100644 --- a/internal/filter/packagename_flag.go +++ b/internal/filter/packagename_flag.go @@ -2,17 +2,17 @@ package filter import ( "fmt" - "regexp" "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/packages" + "github.com/gobwas/glob" "github.com/spf13/cobra" ) type PackageNameFlag struct { FilterFlagBase - patterns []*regexp.Regexp + patterns []glob.Glob } func (f *PackageNameFlag) Parse(cmd *cobra.Command) error { @@ -21,13 +21,13 @@ func (f *PackageNameFlag) Parse(cmd *cobra.Command) error { return cobraext.FlagParsingError(err, cobraext.FilterPackagesFlagName) } - patternStrings := splitAndTrim(packageNamePatterns, ",") - for patternString := range patternStrings { - regex, err := regexp.Compile(patternString) + patterns := splitAndTrim(packageNamePatterns, ",") + for patternString := range patterns { + pattern, err := glob.Compile(patternString) if err != nil { return fmt.Errorf("invalid package name pattern: %s: %w", patternString, err) } - f.patterns = append(f.patterns, regex) + f.patterns = append(f.patterns, pattern) } if len(f.patterns) > 0 { @@ -43,7 +43,7 @@ func (f *PackageNameFlag) Validate() error { func (f *PackageNameFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { for _, pattern := range f.patterns { - if pattern.MatchString(pkgDirName) { + if pattern.Match(pkgDirName) { return true } } From b10bb7f74d2976ee959ee08daaa4f68ba9c95f5d Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Fri, 24 Oct 2025 13:44:50 -0400 Subject: [PATCH 30/36] removing duplicate code because I overlooked it while merging branches. --- internal/packages/packages.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/internal/packages/packages.go b/internal/packages/packages.go index d31fef51c3..d7f91a0b78 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -722,21 +722,3 @@ func isIntegrationRepo(path string) (bool, error) { return true, nil } - -func ListPackages(root string) ([]PackageManifest, error) { - files, err := filepath.Glob(filepath.Join(root, "packages", "*", PackageManifestFile)) - if err != nil { - return nil, fmt.Errorf("failed matching files with package manifests: %w", err) - } - - var packages []PackageManifest - for _, file := range files { - manifest, err := ReadPackageManifest(file) - if err != nil { - return nil, fmt.Errorf("failed to read package manifest: %w", err) - } - packages = append(packages, *manifest) - } - - return packages, nil -} From ab72763631115a240c685a48b3ed4d48b9de5bea Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Fri, 24 Oct 2025 13:57:01 -0400 Subject: [PATCH 31/36] move allowlist out of the validateSubCommand func --- cmd/foreach.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/cmd/foreach.go b/cmd/foreach.go index c9c01f98eb..80d954ba3b 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -30,6 +30,20 @@ any elastic-package subcommand across multiple packages in a single operation. The command uses the same filter flags as the 'filter' command to select packages, then executes the specified subcommand for each matched package.` +// getAllowedSubCommands returns the list of allowed subcommands for the foreach command. +func getAllowedSubCommands() []string { + return []string{ + "build", + "check", + "clean", + "format", + "install", + "lint", + "test", + "uninstall", + } +} + func setupForeachCommand() *cobraext.Command { cmd := &cobra.Command{ Use: "foreach [flags] -- ", @@ -139,19 +153,8 @@ func executeCommand(args []string, path string) error { } func validateSubCommand(subCommand string) error { - allowedSubCommands := []string{ - "build", - "check", - "clean", - "format", - "install", - "lint", - "test", - "uninstall", - } - - if !slices.Contains(allowedSubCommands, subCommand) { - return fmt.Errorf("invalid subcommand: %s. Allowed subcommands are: %s", subCommand, strings.Join(allowedSubCommands, ", ")) + if !slices.Contains(getAllowedSubCommands(), subCommand) { + return fmt.Errorf("invalid subcommand: %s. Allowed subcommands are: %s", subCommand, strings.Join(getAllowedSubCommands(), ", ")) } return nil From 005c2c33e8017eb3c1a0d99bb29a1fd89ad1bf71 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Fri, 24 Oct 2025 15:27:43 -0400 Subject: [PATCH 32/36] uniform command flag name. --- internal/cobraext/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index c2c8b31b6e..8e6e1bc0eb 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -155,8 +155,8 @@ const ( FilterOutputPackageNameFlagShorthand = "p" FilterOutputPackageNameFlagDescription = "print the package name instead of the directory name in the output" - FilterPackagesFlagName = "packages" - FilterPackagesFlagDescription = "regex pattern to filter packages by" + FilterPackagesFlagName = "package-name" + FilterPackagesFlagDescription = "package names to filter by (comma-separated values)" FilterSpecVersionFlagName = "spec-version" FilterSpecVersionFlagDescription = "Package spec version to filter by (semver)" From 8189930a0d88227dc65822831a2cfc205578d2dc Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Mon, 27 Oct 2025 11:15:28 -0400 Subject: [PATCH 33/36] create a type to hold dirname and manifest --- cmd/filter.go | 12 ++++++------ cmd/foreach.go | 4 ++-- internal/filter/category_flag.go | 14 +++++++------- internal/filter/codeowner_flag.go | 14 +++++++------- internal/filter/input_flag.go | 14 +++++++------- internal/filter/integrationtype_flag.go | 15 +++++++-------- internal/filter/packagename_flag.go | 14 +++++++------- internal/filter/registry.go | 2 +- internal/filter/specversion_flag.go | 15 +++++++-------- internal/filter/type.go | 6 +++--- internal/filter/utils.go | 6 +++--- internal/packages/packages.go | 15 ++++++++++++--- 12 files changed, 69 insertions(+), 62 deletions(-) diff --git a/cmd/filter.go b/cmd/filter.go index 9f1e83e7f4..a6edf55568 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -55,7 +55,7 @@ func filterCommandAction(cmd *cobra.Command, args []string) error { return nil } -func filterPackage(cmd *cobra.Command) (map[string]packages.PackageManifest, error) { +func filterPackage(cmd *cobra.Command) ([]packages.PackageDirNameAndManifest, error) { filters := filter.NewFilterRegistry() if err := filters.Parse(cmd); err != nil { @@ -74,7 +74,7 @@ func filterPackage(cmd *cobra.Command) (map[string]packages.PackageManifest, err return filtered, nil } -func printPkgList(pkgs map[string]packages.PackageManifest, printPackageName bool, w io.Writer) error { +func printPkgList(pkgs []packages.PackageDirNameAndManifest, printPackageName bool, w io.Writer) error { enc := json.NewEncoder(w) enc.SetEscapeHTML(false) if len(pkgs) == 0 { @@ -83,12 +83,12 @@ func printPkgList(pkgs map[string]packages.PackageManifest, printPackageName boo names := make([]string, 0, len(pkgs)) if printPackageName { - for _, pkgManifest := range pkgs { - names = append(names, pkgManifest.Name) + for _, pkg := range pkgs { + names = append(names, pkg.Manifest.Name) } } else { - for pkgDirName := range pkgs { - names = append(names, pkgDirName) + for _, pkg := range pkgs { + names = append(names, pkg.DirName) } } diff --git a/cmd/foreach.go b/cmd/foreach.go index 80d954ba3b..d6bab71f33 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -112,8 +112,8 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { }(packageChan) } - for pkgName := range filtered { - packageChan <- pkgName + for _, pkg := range filtered { + packageChan <- pkg.DirName } close(packageChan) diff --git a/internal/filter/category_flag.go b/internal/filter/category_flag.go index 8361e99a3e..e49f488ae1 100644 --- a/internal/filter/category_flag.go +++ b/internal/filter/category_flag.go @@ -30,15 +30,15 @@ func (f *CategoryFlag) Validate() error { return nil } -func (f *CategoryFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { - return hasAnyMatch(f.values, pkgManifest.Categories) +func (f *CategoryFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + return hasAnyMatch(f.values, manifest.Categories) } -func (f *CategoryFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) { - filtered := make(map[string]packages.PackageManifest, len(pkgs)) - for pkgDirName, pkgManifest := range pkgs { - if f.Matches(pkgDirName, pkgManifest) { - filtered[pkgDirName] = pkgManifest +func (f *CategoryFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) } } return filtered, nil diff --git a/internal/filter/codeowner_flag.go b/internal/filter/codeowner_flag.go index 417c153b86..6d8c4123e9 100644 --- a/internal/filter/codeowner_flag.go +++ b/internal/filter/codeowner_flag.go @@ -42,15 +42,15 @@ func (f *CodeOwnerFlag) Validate() error { return nil } -func (f *CodeOwnerFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { - return hasAnyMatch(f.values, []string{pkgManifest.Owner.Github}) +func (f *CodeOwnerFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + return hasAnyMatch(f.values, []string{manifest.Owner.Github}) } -func (f *CodeOwnerFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) { - filtered := make(map[string]packages.PackageManifest, len(pkgs)) - for pkgDirName, pkgManifest := range pkgs { - if f.Matches(pkgDirName, pkgManifest) { - filtered[pkgDirName] = pkgManifest +func (f *CodeOwnerFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) } } return filtered, nil diff --git a/internal/filter/input_flag.go b/internal/filter/input_flag.go index 57726fdf99..f94b7b42eb 100644 --- a/internal/filter/input_flag.go +++ b/internal/filter/input_flag.go @@ -31,9 +31,9 @@ func (f *InputFlag) Validate() error { return nil } -func (f *InputFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { +func (f *InputFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { if f.values != nil { - inputs := extractInputs(pkgManifest) + inputs := extractInputs(manifest) if !hasAnyMatch(f.values, inputs) { return false } @@ -41,12 +41,12 @@ func (f *InputFlag) Matches(pkgDirName string, pkgManifest packages.PackageManif return true } -func (f *InputFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) { - filtered := make(map[string]packages.PackageManifest, len(pkgs)) +func (f *InputFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) - for pkgName, pkgManifest := range pkgs { - if f.Matches(pkgName, pkgManifest) { - filtered[pkgName] = pkgManifest + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) } } return filtered, nil diff --git a/internal/filter/integrationtype_flag.go b/internal/filter/integrationtype_flag.go index 819617a3ec..3c400ed2a5 100644 --- a/internal/filter/integrationtype_flag.go +++ b/internal/filter/integrationtype_flag.go @@ -30,21 +30,20 @@ func (f *IntegrationTypeFlag) Validate() error { return nil } -func (f *IntegrationTypeFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { +func (f *IntegrationTypeFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { if f.values != nil { - if !hasAnyMatch(f.values, []string{pkgManifest.Type}) { + if !hasAnyMatch(f.values, []string{manifest.Type}) { return false } } return true } -func (f *IntegrationTypeFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) { - filtered := make(map[string]packages.PackageManifest, len(pkgs)) - - for pkgName, pkgManifest := range pkgs { - if f.Matches(pkgName, pkgManifest) { - filtered[pkgName] = pkgManifest +func (f *IntegrationTypeFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) } } return filtered, nil diff --git a/internal/filter/packagename_flag.go b/internal/filter/packagename_flag.go index 478f27674a..6dbb435471 100644 --- a/internal/filter/packagename_flag.go +++ b/internal/filter/packagename_flag.go @@ -41,20 +41,20 @@ func (f *PackageNameFlag) Validate() error { return nil } -func (f *PackageNameFlag) Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool { +func (f *PackageNameFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { for _, pattern := range f.patterns { - if pattern.Match(pkgDirName) { + if pattern.Match(dirName) { return true } } return false } -func (f *PackageNameFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) { - filtered := make(map[string]packages.PackageManifest, len(pkgs)) - for pkgDirName, pkgManifest := range pkgs { - if f.Matches(pkgDirName, pkgManifest) { - filtered[pkgDirName] = pkgManifest +func (f *PackageNameFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) } } return filtered, nil diff --git a/internal/filter/registry.go b/internal/filter/registry.go index 6dc5b6ad11..fb4409149d 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -65,7 +65,7 @@ func (r *FilterRegistry) Validate() error { return nil } -func (r *FilterRegistry) Execute() (filtered map[string]packages.PackageManifest, errors multierror.Error) { +func (r *FilterRegistry) Execute() (filtered []packages.PackageDirNameAndManifest, errors multierror.Error) { root, err := packages.MustFindIntegrationRoot() if err != nil { return nil, multierror.Error{err} diff --git a/internal/filter/specversion_flag.go b/internal/filter/specversion_flag.go index deb9d134c4..dd785b8c19 100644 --- a/internal/filter/specversion_flag.go +++ b/internal/filter/specversion_flag.go @@ -40,20 +40,19 @@ func (f *SpecVersionFlag) Validate() error { return nil } -func (f *SpecVersionFlag) Matches(pkgName string, pkg packages.PackageManifest) bool { - pkgVersion, err := semver.NewVersion(pkg.SpecVersion) +func (f *SpecVersionFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + pkgVersion, err := semver.NewVersion(manifest.SpecVersion) if err != nil { return false } return f.constraints.Check(pkgVersion) } -func (f *SpecVersionFlag) ApplyTo(pkgs map[string]packages.PackageManifest) (filtered map[string]packages.PackageManifest, err error) { - filtered = make(map[string]packages.PackageManifest, len(pkgs)) - - for pkgName, pkg := range pkgs { - if f.Matches(pkgName, pkg) { - filtered[pkgName] = pkg +func (f *SpecVersionFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) } } return filtered, nil diff --git a/internal/filter/type.go b/internal/filter/type.go index ef191478df..1a4e02f701 100644 --- a/internal/filter/type.go +++ b/internal/filter/type.go @@ -23,10 +23,10 @@ type Filter interface { Parse(cmd *cobra.Command) error Validate() error - ApplyTo(pkgs map[string]packages.PackageManifest) (map[string]packages.PackageManifest, error) + ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) // Matches checks if a package matches the filter criteria. - // pkgDirName is the directory name of the package in package root. - Matches(pkgDirName string, pkgManifest packages.PackageManifest) bool + // dirName is the directory name of the package in package root. + Matches(dirName string, manifest *packages.PackageManifest) bool } // FilterFlagBase provides common functionality for filter flags. diff --git a/internal/filter/utils.go b/internal/filter/utils.go index eb6163c3f0..0c297e4ade 100644 --- a/internal/filter/utils.go +++ b/internal/filter/utils.go @@ -38,10 +38,9 @@ func hasAnyMatch(filters map[string]struct{}, items []string) bool { } // extractInputs extracts all input types from package policy templates -func extractInputs(pkg packages.PackageManifest) []string { - var inputs []string +func extractInputs(manifest *packages.PackageManifest) []string { uniqueInputs := make(map[string]struct{}) - for _, policyTemplate := range pkg.PolicyTemplates { + for _, policyTemplate := range manifest.PolicyTemplates { if policyTemplate.Input != "" { uniqueInputs[policyTemplate.Input] = struct{}{} } @@ -51,6 +50,7 @@ func extractInputs(pkg packages.PackageManifest) []string { } } + inputs := make([]string, 0, len(uniqueInputs)) for input := range uniqueInputs { inputs = append(inputs, input) } diff --git a/internal/packages/packages.go b/internal/packages/packages.go index d7f91a0b78..74577a6d0c 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -204,6 +204,11 @@ type PackageManifest struct { Elasticsearch *Elasticsearch `config:"elasticsearch" json:"elasticsearch" yaml:"elasticsearch"` } +type PackageDirNameAndManifest struct { + DirName string + Manifest *PackageManifest +} + type ManifestIndexTemplate struct { IngestPipeline *ManifestIngestPipeline `config:"ingest_pipeline" json:"ingest_pipeline" yaml:"ingest_pipeline"` Mappings *ManifestMappings `config:"mappings" json:"mappings" yaml:"mappings"` @@ -471,20 +476,24 @@ func ReadPackageManifest(path string) (*PackageManifest, error) { } // ReadAllPackageManifests reads all the package manifests in the given root directory. -func ReadAllPackageManifests(root string) (map[string]PackageManifest, error) { +func ReadAllPackageManifests(root string) ([]PackageDirNameAndManifest, error) { files, err := filepath.Glob(filepath.Join(root, "packages", "*", PackageManifestFile)) if err != nil { return nil, fmt.Errorf("failed matching files with package manifests: %w", err) } - packages := make(map[string]PackageManifest, len(files)) + packages := make([]PackageDirNameAndManifest, 0, len(files)) for _, file := range files { + dirName := filepath.Base(filepath.Dir(file)) manifest, err := ReadPackageManifest(file) if err != nil { return nil, fmt.Errorf("failed to read package manifest: %w", err) } - packages[filepath.Base(filepath.Dir(file))] = *manifest + packages = append(packages, PackageDirNameAndManifest{ + DirName: dirName, + Manifest: manifest, + }) } return packages, nil From caaff9abcc9abb60ec525e40eab441ffd99b7828 Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Mon, 27 Oct 2025 11:44:34 -0400 Subject: [PATCH 34/36] lint and build docs --- README.md | 18 ++++++++++++++++++ cmd/filter.go | 3 ++- cmd/foreach.go | 3 ++- internal/filter/category_flag.go | 7 ++++++- internal/filter/codeowner_flag.go | 7 ++++++- internal/filter/input_flag.go | 7 ++++++- internal/filter/integrationtype_flag.go | 7 ++++++- internal/filter/packagename_flag.go | 9 +++++++-- internal/filter/registry.go | 7 ++++++- internal/filter/specversion_flag.go | 7 ++++++- internal/filter/type.go | 7 ++++++- internal/filter/utils.go | 4 ++++ 12 files changed, 75 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index e23361b7a1..4003076a66 100644 --- a/README.md +++ b/README.md @@ -366,6 +366,24 @@ Use this command to export ingest pipelines with referenced pipelines from the E Use this command to download selected ingest pipelines and its referenced processor pipelines from Elasticsearch. Select data stream or the package root directories to download the pipelines. Pipelines are downloaded as is and will need adjustment to meet your package needs. +### `elastic-package filter [flags]` + +_Context: package_ + +This command gives you a list of all packages based on the given query + +### `elastic-package foreach [flags] -- ` + +_Context: package_ + +Execute a command for each package matching the given filter criteria. + +This command combines filtering capabilities with command execution, allowing you to run +any elastic-package subcommand across multiple packages in a single operation. + +The command uses the same filter flags as the 'filter' command to select packages, +then executes the specified subcommand for each matched package. + ### `elastic-package format` _Context: package_ diff --git a/cmd/filter.go b/cmd/filter.go index a6edf55568..655963541c 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -11,10 +11,11 @@ import ( "os" "slices" + "github.com/spf13/cobra" + "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/filter" "github.com/elastic/elastic-package/internal/packages" - "github.com/spf13/cobra" ) const filterLongDescription = `This command gives you a list of all packages based on the given query` diff --git a/cmd/foreach.go b/cmd/foreach.go index d6bab71f33..95f81bf492 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -14,12 +14,13 @@ import ( "strings" "sync" + "github.com/spf13/cobra" + "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/filter" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" - "github.com/spf13/cobra" ) const foreachLongDescription = `Execute a command for each package matching the given filter criteria. diff --git a/internal/filter/category_flag.go b/internal/filter/category_flag.go index e49f488ae1..143d13caa8 100644 --- a/internal/filter/category_flag.go +++ b/internal/filter/category_flag.go @@ -1,9 +1,14 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package filter import ( + "github.com/spf13/cobra" + "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/packages" - "github.com/spf13/cobra" ) type CategoryFlag struct { diff --git a/internal/filter/codeowner_flag.go b/internal/filter/codeowner_flag.go index 6d8c4123e9..06e531084a 100644 --- a/internal/filter/codeowner_flag.go +++ b/internal/filter/codeowner_flag.go @@ -1,12 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package filter import ( "fmt" + "github.com/spf13/cobra" + "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/tui" - "github.com/spf13/cobra" ) type CodeOwnerFlag struct { diff --git a/internal/filter/input_flag.go b/internal/filter/input_flag.go index f94b7b42eb..5c4479380f 100644 --- a/internal/filter/input_flag.go +++ b/internal/filter/input_flag.go @@ -1,9 +1,14 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package filter import ( + "github.com/spf13/cobra" + "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/packages" - "github.com/spf13/cobra" ) type InputFlag struct { diff --git a/internal/filter/integrationtype_flag.go b/internal/filter/integrationtype_flag.go index 3c400ed2a5..d1bbc7934f 100644 --- a/internal/filter/integrationtype_flag.go +++ b/internal/filter/integrationtype_flag.go @@ -1,9 +1,14 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package filter import ( + "github.com/spf13/cobra" + "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/packages" - "github.com/spf13/cobra" ) type IntegrationTypeFlag struct { diff --git a/internal/filter/packagename_flag.go b/internal/filter/packagename_flag.go index 6dbb435471..0d5776cab0 100644 --- a/internal/filter/packagename_flag.go +++ b/internal/filter/packagename_flag.go @@ -1,12 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package filter import ( "fmt" - "github.com/elastic/elastic-package/internal/cobraext" - "github.com/elastic/elastic-package/internal/packages" "github.com/gobwas/glob" "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" ) type PackageNameFlag struct { diff --git a/internal/filter/registry.go b/internal/filter/registry.go index fb4409149d..a0094e84a0 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -1,12 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package filter import ( "fmt" + "github.com/spf13/cobra" + "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" - "github.com/spf13/cobra" ) var registry = []Filter{ diff --git a/internal/filter/specversion_flag.go b/internal/filter/specversion_flag.go index dd785b8c19..c88cc70e6f 100644 --- a/internal/filter/specversion_flag.go +++ b/internal/filter/specversion_flag.go @@ -1,12 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package filter import ( "fmt" "github.com/Masterminds/semver/v3" + "github.com/spf13/cobra" + "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/packages" - "github.com/spf13/cobra" ) type SpecVersionFlag struct { diff --git a/internal/filter/type.go b/internal/filter/type.go index 1a4e02f701..33058b9439 100644 --- a/internal/filter/type.go +++ b/internal/filter/type.go @@ -1,8 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package filter import ( - "github.com/elastic/elastic-package/internal/packages" "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/packages" ) // FilterFlag defines the basic interface for filter flags. diff --git a/internal/filter/utils.go b/internal/filter/utils.go index 0c297e4ade..95cdbdc2f5 100644 --- a/internal/filter/utils.go +++ b/internal/filter/utils.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package filter import ( From f997640c60323b4002c53cb59abd402575cb24ca Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Wed, 5 Nov 2025 09:47:45 -0500 Subject: [PATCH 35/36] solve some review comments --- internal/cobraext/flags.go | 8 +-- .../filter/{category_flag.go => category.go} | 5 +- .../{codeowner_flag.go => codeowner.go} | 4 +- internal/filter/{input_flag.go => input.go} | 2 +- internal/filter/integrationtype_flag.go | 66 ------------------- .../{packagename_flag.go => packagename.go} | 2 +- internal/filter/packagetype.go | 61 +++++++++++++++++ internal/filter/registry.go | 3 +- .../{specversion_flag.go => specversion.go} | 0 internal/filter/type.go | 26 ++------ internal/filter/utils.go | 11 ++-- 11 files changed, 86 insertions(+), 102 deletions(-) rename internal/filter/{category_flag.go => category.go} (94%) rename internal/filter/{codeowner_flag.go => codeowner.go} (96%) rename internal/filter/{input_flag.go => input.go} (98%) delete mode 100644 internal/filter/integrationtype_flag.go rename internal/filter/{packagename_flag.go => packagename.go} (97%) create mode 100644 internal/filter/packagetype.go rename internal/filter/{specversion_flag.go => specversion.go} (100%) diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 8e6e1bc0eb..e6ddf76e41 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -142,8 +142,8 @@ const ( FilterCodeOwnerFlagName = "code-owner" FilterCodeOwnerFlagDescription = "code owners to filter by (comma-separated values)" - FilterIntegrationTypeFlagName = "integration-type" - FilterIntegrationTypeFlagDescription = "integration types to filter by (comma-separated values)" + FilterPackageTypeFlagName = "package-type" + FilterPackageTypeFlagDescription = "package types to filter by (comma-separated values)" FilterInputFlagName = "input" FilterInputFlagDescription = "name of the inputs to filter by (comma-separated values)" @@ -155,14 +155,14 @@ const ( FilterOutputPackageNameFlagShorthand = "p" FilterOutputPackageNameFlagDescription = "print the package name instead of the directory name in the output" - FilterPackagesFlagName = "package-name" + FilterPackagesFlagName = "packages" FilterPackagesFlagDescription = "package names to filter by (comma-separated values)" FilterSpecVersionFlagName = "spec-version" FilterSpecVersionFlagDescription = "Package spec version to filter by (semver)" ForeachPoolSizeFlagName = "parallel" - ForeachPoolSizeFlagDescription = "number of packages to execute in parallel (defaults to serial execution)" + ForeachPoolSizeFlagDescription = "Number of subcommands to execute in parallel (defaults to serial execution)" GenerateTestResultFlagName = "generate" GenerateTestResultFlagDescription = "generate test result file" diff --git a/internal/filter/category_flag.go b/internal/filter/category.go similarity index 94% rename from internal/filter/category_flag.go rename to internal/filter/category.go index 143d13caa8..5c170ca417 100644 --- a/internal/filter/category_flag.go +++ b/internal/filter/category.go @@ -14,7 +14,7 @@ import ( type CategoryFlag struct { FilterFlagBase - values map[string]struct{} + values []string } func (f *CategoryFlag) Parse(cmd *cobra.Command) error { @@ -26,7 +26,8 @@ func (f *CategoryFlag) Parse(cmd *cobra.Command) error { return nil } - f.values = splitAndTrim(category, ",") + categories := splitAndTrim(category, ",") + f.values = categories f.isApplied = true return nil } diff --git a/internal/filter/codeowner_flag.go b/internal/filter/codeowner.go similarity index 96% rename from internal/filter/codeowner_flag.go rename to internal/filter/codeowner.go index 06e531084a..d80b982979 100644 --- a/internal/filter/codeowner_flag.go +++ b/internal/filter/codeowner.go @@ -16,7 +16,7 @@ import ( type CodeOwnerFlag struct { FilterFlagBase - values map[string]struct{} + values []string } func (f *CodeOwnerFlag) Parse(cmd *cobra.Command) error { @@ -37,7 +37,7 @@ func (f *CodeOwnerFlag) Validate() error { validator := tui.Validator{Cwd: "."} if f.values != nil { - for value := range f.values { + for _, value := range f.values { if err := validator.GithubOwner(value); err != nil { return fmt.Errorf("invalid code owner: %s: %w", value, err) } diff --git a/internal/filter/input_flag.go b/internal/filter/input.go similarity index 98% rename from internal/filter/input_flag.go rename to internal/filter/input.go index 5c4479380f..d73f7c3d94 100644 --- a/internal/filter/input_flag.go +++ b/internal/filter/input.go @@ -15,7 +15,7 @@ type InputFlag struct { FilterFlagBase // flag specific fields - values map[string]struct{} + values []string } func (f *InputFlag) Parse(cmd *cobra.Command) error { diff --git a/internal/filter/integrationtype_flag.go b/internal/filter/integrationtype_flag.go deleted file mode 100644 index d1bbc7934f..0000000000 --- a/internal/filter/integrationtype_flag.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package filter - -import ( - "github.com/spf13/cobra" - - "github.com/elastic/elastic-package/internal/cobraext" - "github.com/elastic/elastic-package/internal/packages" -) - -type IntegrationTypeFlag struct { - FilterFlagBase - - // flag specific fields - values map[string]struct{} -} - -func (f *IntegrationTypeFlag) Parse(cmd *cobra.Command) error { - integrationTypes, err := cmd.Flags().GetString(cobraext.FilterIntegrationTypeFlagName) - if err != nil { - return cobraext.FlagParsingError(err, cobraext.FilterIntegrationTypeFlagName) - } - if integrationTypes == "" { - return nil - } - f.values = splitAndTrim(integrationTypes, ",") - f.isApplied = true - return nil -} - -func (f *IntegrationTypeFlag) Validate() error { - return nil -} - -func (f *IntegrationTypeFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { - if f.values != nil { - if !hasAnyMatch(f.values, []string{manifest.Type}) { - return false - } - } - return true -} - -func (f *IntegrationTypeFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { - filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) - for _, pkg := range pkgs { - if f.Matches(pkg.DirName, pkg.Manifest) { - filtered = append(filtered, pkg) - } - } - return filtered, nil -} - -func initIntegrationTypeFlag() *IntegrationTypeFlag { - return &IntegrationTypeFlag{ - FilterFlagBase: FilterFlagBase{ - name: cobraext.FilterIntegrationTypeFlagName, - description: cobraext.FilterIntegrationTypeFlagDescription, - shorthand: "", - defaultValue: "", - }, - } -} diff --git a/internal/filter/packagename_flag.go b/internal/filter/packagename.go similarity index 97% rename from internal/filter/packagename_flag.go rename to internal/filter/packagename.go index 0d5776cab0..164e6753c4 100644 --- a/internal/filter/packagename_flag.go +++ b/internal/filter/packagename.go @@ -27,7 +27,7 @@ func (f *PackageNameFlag) Parse(cmd *cobra.Command) error { } patterns := splitAndTrim(packageNamePatterns, ",") - for patternString := range patterns { + for _, patternString := range patterns { pattern, err := glob.Compile(patternString) if err != nil { return fmt.Errorf("invalid package name pattern: %s: %w", patternString, err) diff --git a/internal/filter/packagetype.go b/internal/filter/packagetype.go new file mode 100644 index 0000000000..d1945a4b2f --- /dev/null +++ b/internal/filter/packagetype.go @@ -0,0 +1,61 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type PackageTypeFlag struct { + FilterFlagBase + + // flag specific fields + values []string +} + +func (f *PackageTypeFlag) Parse(cmd *cobra.Command) error { + packageTypes, err := cmd.Flags().GetString(cobraext.FilterPackageTypeFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterPackageTypeFlagName) + } + if packageTypes == "" { + return nil + } + f.values = splitAndTrim(packageTypes, ",") + f.isApplied = true + return nil +} + +func (f *PackageTypeFlag) Validate() error { + return nil +} + +func (f *PackageTypeFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + return hasAnyMatch(f.values, []string{manifest.Type}) +} + +func (f *PackageTypeFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initPackageTypeFlag() *PackageTypeFlag { + return &PackageTypeFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterPackageTypeFlagName, + description: cobraext.FilterPackageTypeFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/registry.go b/internal/filter/registry.go index a0094e84a0..63e06ce8e1 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -18,8 +18,8 @@ var registry = []Filter{ initCategoryFlag(), initCodeOwnerFlag(), initInputFlag(), - initIntegrationTypeFlag(), initPackageNameFlag(), + initPackageTypeFlag(), initSpecVersionFlag(), } @@ -83,6 +83,7 @@ func (r *FilterRegistry) Execute() (filtered []packages.PackageDirNameAndManifes filtered = pkgs for _, filter := range r.filters { + logger.Infof("Applying for %d packages", len(filtered)) filtered, err = filter.ApplyTo(filtered) if err != nil { errors = append(errors, err) diff --git a/internal/filter/specversion_flag.go b/internal/filter/specversion.go similarity index 100% rename from internal/filter/specversion_flag.go rename to internal/filter/specversion.go diff --git a/internal/filter/type.go b/internal/filter/type.go index 33058b9439..90bd2ea391 100644 --- a/internal/filter/type.go +++ b/internal/filter/type.go @@ -5,6 +5,8 @@ package filter import ( + "fmt" + "github.com/spf13/cobra" "github.com/elastic/elastic-package/internal/packages" @@ -12,11 +14,7 @@ import ( // FilterFlag defines the basic interface for filter flags. type FilterFlag interface { - Name() string - Description() string - Shorthand() string - DefaultValue() string - + String() string Register(cmd *cobra.Command) IsApplied() bool } @@ -44,24 +42,12 @@ type FilterFlagBase struct { isApplied bool } -func (f *FilterFlagBase) Name() string { - return f.name -} - -func (f *FilterFlagBase) Description() string { - return f.description -} - -func (f *FilterFlagBase) Shorthand() string { - return f.shorthand -} - -func (f *FilterFlagBase) DefaultValue() string { - return f.defaultValue +func (f *FilterFlagBase) String() string { + return fmt.Sprintf("name=%s defaultValue=%s applied=%v", f.name, f.defaultValue, f.isApplied) } func (f *FilterFlagBase) Register(cmd *cobra.Command) { - cmd.Flags().StringP(f.Name(), f.Shorthand(), f.DefaultValue(), f.Description()) + cmd.Flags().StringP(f.name, f.shorthand, f.defaultValue, f.description) } func (f *FilterFlagBase) IsApplied() bool { diff --git a/internal/filter/utils.go b/internal/filter/utils.go index 95cdbdc2f5..d1b416a019 100644 --- a/internal/filter/utils.go +++ b/internal/filter/utils.go @@ -5,35 +5,36 @@ package filter import ( + "slices" "strings" "github.com/elastic/elastic-package/internal/packages" ) // splitAndTrim splits a string by delimiter and trims whitespace from each element -func splitAndTrim(s, delimiter string) map[string]struct{} { +func splitAndTrim(s, delimiter string) []string { if s == "" { return nil } parts := strings.Split(s, delimiter) - result := make(map[string]struct{}, len(parts)) + result := make([]string, 0, len(parts)) for _, part := range parts { trimmed := strings.TrimSpace(part) if trimmed != "" { - result[trimmed] = struct{}{} + result = append(result, trimmed) } } return result } // hasAnyMatch checks if any item in the items slice exists in the filters slice -func hasAnyMatch(filters map[string]struct{}, items []string) bool { +func hasAnyMatch(filters []string, items []string) bool { if len(filters) == 0 { return true } for _, item := range items { - if _, ok := filters[item]; ok { + if slices.Contains(filters, item) { return true } } From a9699e4ebec112554873afc4dab7e94ef39daeee Mon Sep 17 00:00:00 2001 From: vinit-chauhan Date: Thu, 6 Nov 2025 13:48:53 -0500 Subject: [PATCH 36/36] - add way to filter packages in arbitrary directory. - update --packages to use only package name; added another filter to filter by package dirs. - Added auto discovery of package for configurable depth. - Added flag to exclude dirs from filter process. --- cmd/filter.go | 30 +++++-- cmd/foreach.go | 27 ++---- internal/cobraext/flags.go | 24 +++-- internal/filter/packagedirname.go | 77 ++++++++++++++++ internal/filter/packagename.go | 2 +- internal/filter/registry.go | 22 +++-- internal/packages/packages.go | 145 ++++++++++++++---------------- 7 files changed, 215 insertions(+), 112 deletions(-) create mode 100644 internal/filter/packagedirname.go diff --git a/cmd/filter.go b/cmd/filter.go index 655963541c..5944907615 100644 --- a/cmd/filter.go +++ b/cmd/filter.go @@ -32,8 +32,9 @@ func setupFilterCommand() *cobraext.Command { // add filter flags to the command (input, code owner, kibana version, categories) filter.SetFilterFlags(cmd) - // add the output package name flag to the command - cmd.Flags().BoolP(cobraext.FilterOutputPackageNameFlagName, cobraext.FilterOutputPackageNameFlagShorthand, false, cobraext.FilterOutputPackageNameFlagDescription) + // add the output package name and absolute path flags to the command + cmd.Flags().BoolP(cobraext.FilterOutputPackageNameFlagName, "", false, cobraext.FilterOutputPackageNameFlagDescription) + cmd.Flags().BoolP(cobraext.FilterOutputAbsolutePathFlagName, "", false, cobraext.FilterOutputAbsolutePathFlagDescription) return cobraext.NewCommand(cmd, cobraext.ContextPackage) } @@ -49,7 +50,12 @@ func filterCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("getting output package name flag failed: %w", err) } - if err = printPkgList(filtered, printPackageName, os.Stdout); err != nil { + outputAbsolutePath, err := cmd.Flags().GetBool("output-absolute-path") + if err != nil { + return fmt.Errorf("getting output absolute path flag failed: %w", err) + } + + if err = printPkgList(filtered, printPackageName, outputAbsolutePath, os.Stdout); err != nil { return fmt.Errorf("printing JSON failed: %w", err) } @@ -57,7 +63,17 @@ func filterCommandAction(cmd *cobra.Command, args []string) error { } func filterPackage(cmd *cobra.Command) ([]packages.PackageDirNameAndManifest, error) { - filters := filter.NewFilterRegistry() + depth, err := cmd.Flags().GetInt(cobraext.FilterDepthFlagName) + if err != nil { + return nil, fmt.Errorf("getting depth flag failed: %w", err) + } + + excludeDirs, err := cmd.Flags().GetString(cobraext.FilterExcludeDirFlagName) + if err != nil { + return nil, fmt.Errorf("getting exclude-dir flag failed: %w", err) + } + + filters := filter.NewFilterRegistry(depth, excludeDirs) if err := filters.Parse(cmd); err != nil { return nil, fmt.Errorf("parsing filter options failed: %w", err) @@ -75,7 +91,7 @@ func filterPackage(cmd *cobra.Command) ([]packages.PackageDirNameAndManifest, er return filtered, nil } -func printPkgList(pkgs []packages.PackageDirNameAndManifest, printPackageName bool, w io.Writer) error { +func printPkgList(pkgs []packages.PackageDirNameAndManifest, printPackageName bool, outputAbsolutePath bool, w io.Writer) error { enc := json.NewEncoder(w) enc.SetEscapeHTML(false) if len(pkgs) == 0 { @@ -87,6 +103,10 @@ func printPkgList(pkgs []packages.PackageDirNameAndManifest, printPackageName bo for _, pkg := range pkgs { names = append(names, pkg.Manifest.Name) } + } else if outputAbsolutePath { + for _, pkg := range pkgs { + names = append(names, pkg.Path) + } } else { for _, pkg := range pkgs { names = append(names, pkg.DirName) diff --git a/cmd/foreach.go b/cmd/foreach.go index 95f81bf492..d994b7b50d 100644 --- a/cmd/foreach.go +++ b/cmd/foreach.go @@ -9,7 +9,6 @@ import ( "io" "os" "os/exec" - "path/filepath" "slices" "strings" "sync" @@ -20,7 +19,6 @@ import ( "github.com/elastic/elastic-package/internal/filter" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" - "github.com/elastic/elastic-package/internal/packages" ) const foreachLongDescription = `Execute a command for each package matching the given filter criteria. @@ -60,7 +58,7 @@ func setupForeachCommand() *cobraext.Command { filter.SetFilterFlags(cmd) // Add pool size flag - cmd.Flags().IntP(cobraext.ForeachPoolSizeFlagName, "", 1, cobraext.ForeachPoolSizeFlagDescription) + cmd.Flags().IntP(cobraext.ForeachPoolSizeFlagName, cobraext.ForeachPoolSizeFlagShorthand, 1, cobraext.ForeachPoolSizeFlagDescription) return cobraext.NewCommand(cmd, cobraext.ContextPackage) } @@ -75,12 +73,6 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { return fmt.Errorf("validating sub command failed: %w", err) } - // Find integration root - root, err := packages.MustFindIntegrationRoot() - if err != nil { - return fmt.Errorf("can't find integration root: %w", err) - } - // reuse filterPackage from cmd/filter.go filtered, err := filterPackage(cmd) if err != nil { @@ -92,31 +84,30 @@ func foreachCommandAction(cmd *cobra.Command, args []string) error { errs := multierror.Error{} successes := 0 - packageChan := make(chan string, poolSize) + packagePathChan := make(chan string, poolSize) for range poolSize { wg.Add(1) - go func(packageChan <-chan string) { + go func(packagePathChan <-chan string) { defer wg.Done() - for packageName := range packageChan { - path := filepath.Join(root, "packages", packageName) - err := executeCommand(args, path) + for packagePath := range packagePathChan { + err := executeCommand(args, packagePath) mu.Lock() if err != nil { - errs = append(errs, fmt.Errorf("executing command for package %s failed: %w", packageName, err)) + errs = append(errs, fmt.Errorf("executing command for package %s failed: %w", packagePath, err)) } else { successes++ } mu.Unlock() } - }(packageChan) + }(packagePathChan) } for _, pkg := range filtered { - packageChan <- pkg.DirName + packagePathChan <- pkg.Path } - close(packageChan) + close(packagePathChan) wg.Wait() diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index e6ddf76e41..e5fc173abe 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -136,32 +136,46 @@ const ( FailFastFlagName = "fail-fast" FailFastFlagDescription = "fail immediately if any file requires updates (do not overwrite)" - FilterCategoriesFlagName = "category" + FilterCategoriesFlagName = "categories" FilterCategoriesFlagDescription = "integration categories to filter by (comma-separated values)" - FilterCodeOwnerFlagName = "code-owner" + FilterCodeOwnerFlagName = "code-owners" FilterCodeOwnerFlagDescription = "code owners to filter by (comma-separated values)" - FilterPackageTypeFlagName = "package-type" + FilterPackageTypeFlagName = "package-types" FilterPackageTypeFlagDescription = "package types to filter by (comma-separated values)" - FilterInputFlagName = "input" + FilterInputFlagName = "inputs" FilterInputFlagDescription = "name of the inputs to filter by (comma-separated values)" FilterKibanaVersionFlagName = "kibana-version" FilterKibanaVersionFlagDescription = "kibana version to filter by (semver)" + FilterOutputAbsolutePathFlagName = "output-absolute-path" + FilterOutputAbsolutePathFlagDescription = "output the absolute path of the package" + FilterOutputPackageNameFlagName = "output-package-name" - FilterOutputPackageNameFlagShorthand = "p" FilterOutputPackageNameFlagDescription = "print the package name instead of the directory name in the output" + FilterPackageDirNameFlagName = "package-dirs" + FilterPackageDirNameFlagDescription = "package directories to filter by (comma-separated values)" + FilterPackagesFlagName = "packages" FilterPackagesFlagDescription = "package names to filter by (comma-separated values)" FilterSpecVersionFlagName = "spec-version" FilterSpecVersionFlagDescription = "Package spec version to filter by (semver)" + FilterDepthFlagName = "depth" + FilterDepthFlagDescription = "maximum depth to search for packages (default: 2)" + FilterDepthFlagDefault = 2 + FilterDepthFlagShorthand = "d" + + FilterExcludeDirFlagName = "exclude-dirs" + FilterExcludeDirFlagDescription = "comma-separated list of directories to exclude from search" + ForeachPoolSizeFlagName = "parallel" + ForeachPoolSizeFlagShorthand = "p" ForeachPoolSizeFlagDescription = "Number of subcommands to execute in parallel (defaults to serial execution)" GenerateTestResultFlagName = "generate" diff --git a/internal/filter/packagedirname.go b/internal/filter/packagedirname.go new file mode 100644 index 0000000000..32f4a640ed --- /dev/null +++ b/internal/filter/packagedirname.go @@ -0,0 +1,77 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + + "github.com/gobwas/glob" + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type PackageDirNameFlag struct { + FilterFlagBase + + patterns []glob.Glob +} + +func (f *PackageDirNameFlag) Parse(cmd *cobra.Command) error { + packageDirNamePatterns, err := cmd.Flags().GetString(cobraext.FilterPackageDirNameFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterPackageDirNameFlagName) + } + + patterns := splitAndTrim(packageDirNamePatterns, ",") + for _, patternString := range patterns { + pattern, err := glob.Compile(patternString) + if err != nil { + return fmt.Errorf("invalid package dir name pattern: %s: %w", patternString, err) + } + f.patterns = append(f.patterns, pattern) + } + + if len(f.patterns) > 0 { + f.isApplied = true + } + + return nil +} + +func (f *PackageDirNameFlag) Validate() error { + return nil +} + +func (f *PackageDirNameFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + for _, pattern := range f.patterns { + if pattern.Match(dirName) { + return true + } + } + return false +} + +func (f *PackageDirNameFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initPackageDirNameFlag() *PackageDirNameFlag { + return &PackageDirNameFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterPackageDirNameFlagName, + description: cobraext.FilterPackageDirNameFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/packagename.go b/internal/filter/packagename.go index 164e6753c4..63a415f298 100644 --- a/internal/filter/packagename.go +++ b/internal/filter/packagename.go @@ -48,7 +48,7 @@ func (f *PackageNameFlag) Validate() error { func (f *PackageNameFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { for _, pattern := range f.patterns { - if pattern.Match(dirName) { + if pattern.Match(manifest.Name) { return true } } diff --git a/internal/filter/registry.go b/internal/filter/registry.go index 63e06ce8e1..0066a413f3 100644 --- a/internal/filter/registry.go +++ b/internal/filter/registry.go @@ -6,9 +6,11 @@ package filter import ( "fmt" + "os" "github.com/spf13/cobra" + "github.com/elastic/elastic-package/internal/cobraext" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/multierror" "github.com/elastic/elastic-package/internal/packages" @@ -18,6 +20,7 @@ var registry = []Filter{ initCategoryFlag(), initCodeOwnerFlag(), initInputFlag(), + initPackageDirNameFlag(), initPackageNameFlag(), initPackageTypeFlag(), initSpecVersionFlag(), @@ -25,6 +28,9 @@ var registry = []Filter{ // SetFilterFlags registers all filter flags with the given command. func SetFilterFlags(cmd *cobra.Command) { + cmd.Flags().IntP(cobraext.FilterDepthFlagName, cobraext.FilterDepthFlagShorthand, cobraext.FilterDepthFlagDefault, cobraext.FilterDepthFlagDescription) + cmd.Flags().StringP(cobraext.FilterExcludeDirFlagName, "", "", cobraext.FilterExcludeDirFlagDescription) + for _, filterFlag := range registry { filterFlag.Register(cmd) } @@ -32,13 +38,17 @@ func SetFilterFlags(cmd *cobra.Command) { // FilterRegistry manages a collection of filters for package filtering. type FilterRegistry struct { - filters []Filter + filters []Filter + depth int + excludeDirs string } // NewFilterRegistry creates a new FilterRegistry instance. -func NewFilterRegistry() *FilterRegistry { +func NewFilterRegistry(depth int, excludeDirs string) *FilterRegistry { return &FilterRegistry{ - filters: []Filter{}, + filters: []Filter{}, + depth: depth, + excludeDirs: excludeDirs, } } @@ -71,12 +81,12 @@ func (r *FilterRegistry) Validate() error { } func (r *FilterRegistry) Execute() (filtered []packages.PackageDirNameAndManifest, errors multierror.Error) { - root, err := packages.MustFindIntegrationRoot() + currentDir, err := os.Getwd() if err != nil { - return nil, multierror.Error{err} + return nil, multierror.Error{fmt.Errorf("getting current directory failed: %w", err)} } - pkgs, err := packages.ReadAllPackageManifests(root) + pkgs, err := packages.ReadAllPackageManifestsFromRepo(currentDir, r.depth, r.excludeDirs) if err != nil { return nil, multierror.Error{err} } diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 74577a6d0c..42e6b48d90 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -30,9 +30,6 @@ const ( // DataStreamManifestFile is the name of the data stream's manifest file. DataStreamManifestFile = "manifest.yml" - // GoModFile is the name of the go.mod file. - GoModFile = "go.mod" - defaultPipelineName = "default" dataStreamTypeLogs = "logs" @@ -206,6 +203,7 @@ type PackageManifest struct { type PackageDirNameAndManifest struct { DirName string + Path string Manifest *PackageManifest } @@ -341,54 +339,6 @@ func FindPackageRootFrom(fromDir string) (string, bool, error) { return "", false, nil } -// MustFindIntegrationRoot finds and returns the path to the root folder of a package from the working directory. -// It fails with an error if the package root can't be found. -func MustFindIntegrationRoot() (string, error) { - root, found, err := FindIntegrationRoot() - if err != nil { - return "", fmt.Errorf("locating integration root failed: %w", err) - } - if !found { - return "", errors.New("integration root not found") - } - return root, nil -} - -// FindIntegrationRoot finds and returns the path to the root folder of a package from the working directory. -func FindIntegrationRoot() (string, bool, error) { - workDir, err := os.Getwd() - if err != nil { - return "", false, fmt.Errorf("locating working directory failed: %w", err) - } - return FindIntegrationRootFrom(workDir) -} - -// IntegrationRoot function returns the root directory of the integrations -func FindIntegrationRootFrom(fromDir string) (string, bool, error) { - rootDir := filepath.VolumeName(fromDir) + string(filepath.Separator) - - dir := fromDir - for dir != "." { - path := filepath.Join(dir, GoModFile) - fileInfo, err := os.Stat(path) - if err == nil && !fileInfo.IsDir() { - ok, err := isIntegrationRepo(path) - if err != nil { - return "", false, fmt.Errorf("verifying integration repo failed (path: %s): %w", path, err) - } - if ok { - return dir, true, nil - } - } - - if dir == rootDir { - break - } - dir = filepath.Dir(dir) - } - return fromDir, true, nil -} - // FindDataStreamRootForPath finds and returns the path to the root folder of a data stream. func FindDataStreamRootForPath(workDir string) (string, bool, error) { dir := workDir @@ -475,25 +425,82 @@ func ReadPackageManifest(path string) (*PackageManifest, error) { return &m, nil } -// ReadAllPackageManifests reads all the package manifests in the given root directory. -func ReadAllPackageManifests(root string) ([]PackageDirNameAndManifest, error) { - files, err := filepath.Glob(filepath.Join(root, "packages", "*", PackageManifestFile)) - if err != nil { - return nil, fmt.Errorf("failed matching files with package manifests: %w", err) +// ReadAllPackageManifestsFromRepo reads all the package manifests in the given directory. +// It recursively searches for manifest.yml files up to the specified depth. +// - depth: maximum depth to search (1 = current dir + immediate sub dirs) +// - excludeDirs: comma-separated list of directory names to exclude (always excludes .git) +func ReadAllPackageManifestsFromRepo(searchRoot string, depth int, excludeDirs string) ([]PackageDirNameAndManifest, error) { + // Parse exclude directories + excludeMap := map[string]bool{ + ".git": true, // Always exclude .git + } + if excludeDirs != "" { + for dir := range strings.SplitSeq(excludeDirs, ",") { + excludeMap[strings.TrimSpace(dir)] = true + } } - packages := make([]PackageDirNameAndManifest, 0, len(files)) - for _, file := range files { - dirName := filepath.Base(filepath.Dir(file)) - manifest, err := ReadPackageManifest(file) + var packages []PackageDirNameAndManifest + searchRootDepth := strings.Count(searchRoot, string(filepath.Separator)) + + err := filepath.WalkDir(searchRoot, func(path string, d fs.DirEntry, err error) error { if err != nil { - return nil, fmt.Errorf("failed to read package manifest: %w", err) + return err + } + + // Calculate current depth relative to search root + currentDepth := strings.Count(path, string(filepath.Separator)) - searchRootDepth + + // If it's a directory, check if we should skip it + if d.IsDir() { + dirName := d.Name() + + // Skip excluded directories + if excludeMap[dirName] { + return filepath.SkipDir + } + + // Skip if we've exceeded the depth limit (but allow processing the current level) + if currentDepth > depth { + return filepath.SkipDir + } + + return nil + } + + // Check if this is a manifest file + if d.Name() != PackageManifestFile { + return nil + } + + // Validate it's a package manifest + ok, err := isPackageManifest(path) + if err != nil { + // Log the error but continue searching + return nil + } + if !ok { + return nil + } + + // Extract directory name (just the package directory name, not the full path) + dirName := filepath.Base(filepath.Dir(path)) + manifest, err := ReadPackageManifest(path) + if err != nil { + return fmt.Errorf("failed to read package manifest (path: %s): %w", path, err) } packages = append(packages, PackageDirNameAndManifest{ DirName: dirName, Manifest: manifest, + Path: filepath.Dir(path), }) + + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed walking directory tree: %w", err) } return packages, nil @@ -715,19 +722,3 @@ func isDataStreamManifest(path string) (bool, error) { (m.Type == dataStreamTypeLogs || m.Type == dataStreamTypeMetrics || m.Type == dataStreamTypeSynthetics || m.Type == dataStreamTypeTraces), nil } -func isIntegrationRepo(path string) (bool, error) { - modFile, err := os.ReadFile(path) - if err != nil { - return false, fmt.Errorf("reading go.mod file failed: %w", err) - } - - content := string(modFile) - content = strings.SplitN(content, "\n", 2)[0] - content = strings.TrimSpace(content) - - if !strings.HasSuffix(content, "github.com/elastic/integrations") { - return false, fmt.Errorf("integration root %s is not an elastic-package integration", path) - } - - return true, nil -}