diff --git a/.gitignore b/.gitignore index 4aa5e6e60d..28dd2716d3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,9 @@ elastic-package # IDEA .idea +# VSCode +.vscode + # Build directory /build diff --git a/README.md b/README.md index e23361b7a1..4003076a66 100644 --- a/README.md +++ b/README.md @@ -366,6 +366,24 @@ Use this command to export ingest pipelines with referenced pipelines from the E Use this command to download selected ingest pipelines and its referenced processor pipelines from Elasticsearch. Select data stream or the package root directories to download the pipelines. Pipelines are downloaded as is and will need adjustment to meet your package needs. +### `elastic-package filter [flags]` + +_Context: package_ + +This command gives you a list of all packages based on the given query + +### `elastic-package foreach [flags] -- ` + +_Context: package_ + +Execute a command for each package matching the given filter criteria. + +This command combines filtering capabilities with command execution, allowing you to run +any elastic-package subcommand across multiple packages in a single operation. + +The command uses the same filter flags as the 'filter' command to select packages, +then executes the specified subcommand for each matched package. + ### `elastic-package format` _Context: package_ diff --git a/cmd/filter.go b/cmd/filter.go new file mode 100644 index 0000000000..5944907615 --- /dev/null +++ b/cmd/filter.go @@ -0,0 +1,118 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "os" + "slices" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/filter" + "github.com/elastic/elastic-package/internal/packages" +) + +const filterLongDescription = `This command gives you a list of all packages based on the given query` + +func setupFilterCommand() *cobraext.Command { + cmd := &cobra.Command{ + Use: "filter [flags]", + Short: "filter integrations based on given flags", + Long: filterLongDescription, + Args: cobra.NoArgs, + RunE: filterCommandAction, + } + + // add filter flags to the command (input, code owner, kibana version, categories) + filter.SetFilterFlags(cmd) + + // add the output package name and absolute path flags to the command + cmd.Flags().BoolP(cobraext.FilterOutputPackageNameFlagName, "", false, cobraext.FilterOutputPackageNameFlagDescription) + cmd.Flags().BoolP(cobraext.FilterOutputAbsolutePathFlagName, "", false, cobraext.FilterOutputAbsolutePathFlagDescription) + + return cobraext.NewCommand(cmd, cobraext.ContextPackage) +} + +func filterCommandAction(cmd *cobra.Command, args []string) error { + filtered, err := filterPackage(cmd) + if err != nil { + return fmt.Errorf("filtering packages failed: %w", err) + } + + printPackageName, err := cmd.Flags().GetBool(cobraext.FilterOutputPackageNameFlagName) + if err != nil { + return fmt.Errorf("getting output package name flag failed: %w", err) + } + + outputAbsolutePath, err := cmd.Flags().GetBool("output-absolute-path") + if err != nil { + return fmt.Errorf("getting output absolute path flag failed: %w", err) + } + + if err = printPkgList(filtered, printPackageName, outputAbsolutePath, os.Stdout); err != nil { + return fmt.Errorf("printing JSON failed: %w", err) + } + + return nil +} + +func filterPackage(cmd *cobra.Command) ([]packages.PackageDirNameAndManifest, error) { + depth, err := cmd.Flags().GetInt(cobraext.FilterDepthFlagName) + if err != nil { + return nil, fmt.Errorf("getting depth flag failed: %w", err) + } + + excludeDirs, err := cmd.Flags().GetString(cobraext.FilterExcludeDirFlagName) + if err != nil { + return nil, fmt.Errorf("getting exclude-dir flag failed: %w", err) + } + + filters := filter.NewFilterRegistry(depth, excludeDirs) + + if err := filters.Parse(cmd); err != nil { + return nil, fmt.Errorf("parsing filter options failed: %w", err) + } + + if err := filters.Validate(); err != nil { + return nil, fmt.Errorf("validating filter options failed: %w", err) + } + + filtered, errors := filters.Execute() + if errors != nil { + return nil, fmt.Errorf("filtering packages failed: %s", errors.Error()) + } + + return filtered, nil +} + +func printPkgList(pkgs []packages.PackageDirNameAndManifest, printPackageName bool, outputAbsolutePath bool, w io.Writer) error { + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + if len(pkgs) == 0 { + return nil + } + + names := make([]string, 0, len(pkgs)) + if printPackageName { + for _, pkg := range pkgs { + names = append(names, pkg.Manifest.Name) + } + } else if outputAbsolutePath { + for _, pkg := range pkgs { + names = append(names, pkg.Path) + } + } else { + for _, pkg := range pkgs { + names = append(names, pkg.DirName) + } + } + + slices.Sort(names) + return enc.Encode(names) +} diff --git a/cmd/foreach.go b/cmd/foreach.go new file mode 100644 index 0000000000..d994b7b50d --- /dev/null +++ b/cmd/foreach.go @@ -0,0 +1,153 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "fmt" + "io" + "os" + "os/exec" + "slices" + "strings" + "sync" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/filter" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/multierror" +) + +const foreachLongDescription = `Execute a command for each package matching the given filter criteria. + +This command combines filtering capabilities with command execution, allowing you to run +any elastic-package subcommand across multiple packages in a single operation. + +The command uses the same filter flags as the 'filter' command to select packages, +then executes the specified subcommand for each matched package.` + +// getAllowedSubCommands returns the list of allowed subcommands for the foreach command. +func getAllowedSubCommands() []string { + return []string{ + "build", + "check", + "clean", + "format", + "install", + "lint", + "test", + "uninstall", + } +} + +func setupForeachCommand() *cobraext.Command { + cmd := &cobra.Command{ + Use: "foreach [flags] -- ", + Short: "Execute a command for filtered packages", + Long: foreachLongDescription, + Example: ` # Run system tests for packages with specific inputs + elastic-package foreach --input tcp,udp --parallel 10 -- test system -g`, + RunE: foreachCommandAction, + Args: cobra.MinimumNArgs(1), + } + + // Add filter flags + filter.SetFilterFlags(cmd) + + // Add pool size flag + cmd.Flags().IntP(cobraext.ForeachPoolSizeFlagName, cobraext.ForeachPoolSizeFlagShorthand, 1, cobraext.ForeachPoolSizeFlagDescription) + + return cobraext.NewCommand(cmd, cobraext.ContextPackage) +} + +func foreachCommandAction(cmd *cobra.Command, args []string) error { + poolSize, err := cmd.Flags().GetInt(cobraext.ForeachPoolSizeFlagName) + if err != nil { + return fmt.Errorf("getting pool size failed: %w", err) + } + + if err := validateSubCommand(args[0]); err != nil { + return fmt.Errorf("validating sub command failed: %w", err) + } + + // reuse filterPackage from cmd/filter.go + filtered, err := filterPackage(cmd) + if err != nil { + return fmt.Errorf("filtering packages failed: %w", err) + } + + wg := sync.WaitGroup{} + mu := sync.Mutex{} + errs := multierror.Error{} + successes := 0 + + packagePathChan := make(chan string, poolSize) + + for range poolSize { + wg.Add(1) + go func(packagePathChan <-chan string) { + defer wg.Done() + for packagePath := range packagePathChan { + err := executeCommand(args, packagePath) + + mu.Lock() + if err != nil { + errs = append(errs, fmt.Errorf("executing command for package %s failed: %w", packagePath, err)) + } else { + successes++ + } + mu.Unlock() + } + }(packagePathChan) + } + + for _, pkg := range filtered { + packagePathChan <- pkg.Path + } + close(packagePathChan) + + wg.Wait() + + logger.Infof("Successfully executed command for %d packages\n", successes) + logger.Infof("Errors occurred while executing command for %d packages\n", len(errs)) + + if errs.Error() != "" { + return fmt.Errorf("errors occurred while executing command for packages: \n%s", errs.Error()) + } + + return nil +} + +func executeCommand(args []string, path string) error { + // Look up the elastic-package binary in PATH + execPath, err := exec.LookPath("elastic-package") + if err != nil { + return fmt.Errorf("elastic-package binary not found in PATH: %w", err) + } + + cmd := &exec.Cmd{ + Path: execPath, + Args: append([]string{execPath}, args...), + Dir: path, + Stdout: io.Discard, + Stderr: os.Stderr, + Env: os.Environ(), + } + + if err := cmd.Run(); err != nil { + return fmt.Errorf("executing command for package %s failed: %w", path, err) + } + + return nil +} + +func validateSubCommand(subCommand string) error { + if !slices.Contains(getAllowedSubCommands(), subCommand) { + return fmt.Errorf("invalid subcommand: %s. Allowed subcommands are: %s", subCommand, strings.Join(getAllowedSubCommands(), ", ")) + } + + return nil +} diff --git a/cmd/root.go b/cmd/root.go index e449ff5169..ae5ffe2d99 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -26,7 +26,9 @@ var commands = []*cobraext.Command{ setupDumpCommand(), setupEditCommand(), setupExportCommand(), + setupFilterCommand(), setupFormatCommand(), + setupForeachCommand(), setupInstallCommand(), setupLinksCommand(), setupLintCommand(), diff --git a/go.mod b/go.mod index acf569a3cb..346fbe1102 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/elastic/package-spec/v3 v3.5.0 github.com/fatih/color v1.18.0 github.com/go-viper/mapstructure/v2 v2.4.0 + github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.7.0 github.com/google/go-github/v32 v32.1.0 github.com/google/go-querystring v1.1.0 diff --git a/go.sum b/go.sum index b34f5a5d5c..c06a0c3656 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= diff --git a/internal/cobraext/flags.go b/internal/cobraext/flags.go index 1838404fd6..e5fc173abe 100644 --- a/internal/cobraext/flags.go +++ b/internal/cobraext/flags.go @@ -133,8 +133,51 @@ const ( FailOnMissingFlagName = "fail-on-missing" FailOnMissingFlagDescription = "fail if tests are missing" - FailFastFlagName = "fail-fast" - FailFastFlagDescription = "fail immediately if any file requires updates (do not overwrite)" + FailFastFlagName = "fail-fast" + FailFastFlagDescription = "fail immediately if any file requires updates (do not overwrite)" + + FilterCategoriesFlagName = "categories" + FilterCategoriesFlagDescription = "integration categories to filter by (comma-separated values)" + + FilterCodeOwnerFlagName = "code-owners" + FilterCodeOwnerFlagDescription = "code owners to filter by (comma-separated values)" + + FilterPackageTypeFlagName = "package-types" + FilterPackageTypeFlagDescription = "package types to filter by (comma-separated values)" + + FilterInputFlagName = "inputs" + FilterInputFlagDescription = "name of the inputs to filter by (comma-separated values)" + + FilterKibanaVersionFlagName = "kibana-version" + FilterKibanaVersionFlagDescription = "kibana version to filter by (semver)" + + FilterOutputAbsolutePathFlagName = "output-absolute-path" + FilterOutputAbsolutePathFlagDescription = "output the absolute path of the package" + + FilterOutputPackageNameFlagName = "output-package-name" + FilterOutputPackageNameFlagDescription = "print the package name instead of the directory name in the output" + + FilterPackageDirNameFlagName = "package-dirs" + FilterPackageDirNameFlagDescription = "package directories to filter by (comma-separated values)" + + FilterPackagesFlagName = "packages" + FilterPackagesFlagDescription = "package names to filter by (comma-separated values)" + + FilterSpecVersionFlagName = "spec-version" + FilterSpecVersionFlagDescription = "Package spec version to filter by (semver)" + + FilterDepthFlagName = "depth" + FilterDepthFlagDescription = "maximum depth to search for packages (default: 2)" + FilterDepthFlagDefault = 2 + FilterDepthFlagShorthand = "d" + + FilterExcludeDirFlagName = "exclude-dirs" + FilterExcludeDirFlagDescription = "comma-separated list of directories to exclude from search" + + ForeachPoolSizeFlagName = "parallel" + ForeachPoolSizeFlagShorthand = "p" + ForeachPoolSizeFlagDescription = "Number of subcommands to execute in parallel (defaults to serial execution)" + GenerateTestResultFlagName = "generate" GenerateTestResultFlagDescription = "generate test result file" diff --git a/internal/filter/category.go b/internal/filter/category.go new file mode 100644 index 0000000000..5c170ca417 --- /dev/null +++ b/internal/filter/category.go @@ -0,0 +1,62 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type CategoryFlag struct { + FilterFlagBase + + values []string +} + +func (f *CategoryFlag) Parse(cmd *cobra.Command) error { + category, err := cmd.Flags().GetString(cobraext.FilterCategoriesFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterCategoriesFlagName) + } + if category == "" { + return nil + } + + categories := splitAndTrim(category, ",") + f.values = categories + f.isApplied = true + return nil +} + +func (f *CategoryFlag) Validate() error { + return nil +} + +func (f *CategoryFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + return hasAnyMatch(f.values, manifest.Categories) +} + +func (f *CategoryFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initCategoryFlag() *CategoryFlag { + return &CategoryFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterCategoriesFlagName, + description: cobraext.FilterCategoriesFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/codeowner.go b/internal/filter/codeowner.go new file mode 100644 index 0000000000..d80b982979 --- /dev/null +++ b/internal/filter/codeowner.go @@ -0,0 +1,73 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/tui" +) + +type CodeOwnerFlag struct { + FilterFlagBase + values []string +} + +func (f *CodeOwnerFlag) Parse(cmd *cobra.Command) error { + codeOwners, err := cmd.Flags().GetString(cobraext.FilterCodeOwnerFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterCodeOwnerFlagName) + } + if codeOwners == "" { + return nil + } + + f.values = splitAndTrim(codeOwners, ",") + f.isApplied = true + return nil +} + +func (f *CodeOwnerFlag) Validate() error { + validator := tui.Validator{Cwd: "."} + + if f.values != nil { + for _, value := range f.values { + if err := validator.GithubOwner(value); err != nil { + return fmt.Errorf("invalid code owner: %s: %w", value, err) + } + } + } + + return nil +} + +func (f *CodeOwnerFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + return hasAnyMatch(f.values, []string{manifest.Owner.Github}) +} + +func (f *CodeOwnerFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initCodeOwnerFlag() *CodeOwnerFlag { + return &CodeOwnerFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterCodeOwnerFlagName, + description: cobraext.FilterCodeOwnerFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/input.go b/internal/filter/input.go new file mode 100644 index 0000000000..d73f7c3d94 --- /dev/null +++ b/internal/filter/input.go @@ -0,0 +1,69 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type InputFlag struct { + FilterFlagBase + + // flag specific fields + values []string +} + +func (f *InputFlag) Parse(cmd *cobra.Command) error { + input, err := cmd.Flags().GetString(cobraext.FilterInputFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterInputFlagName) + } + if input == "" { + return nil + } + + f.values = splitAndTrim(input, ",") + f.isApplied = true + return nil +} + +func (f *InputFlag) Validate() error { + return nil +} + +func (f *InputFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + if f.values != nil { + inputs := extractInputs(manifest) + if !hasAnyMatch(f.values, inputs) { + return false + } + } + return true +} + +func (f *InputFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initInputFlag() *InputFlag { + return &InputFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterInputFlagName, + description: cobraext.FilterInputFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/packagedirname.go b/internal/filter/packagedirname.go new file mode 100644 index 0000000000..32f4a640ed --- /dev/null +++ b/internal/filter/packagedirname.go @@ -0,0 +1,77 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + + "github.com/gobwas/glob" + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type PackageDirNameFlag struct { + FilterFlagBase + + patterns []glob.Glob +} + +func (f *PackageDirNameFlag) Parse(cmd *cobra.Command) error { + packageDirNamePatterns, err := cmd.Flags().GetString(cobraext.FilterPackageDirNameFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterPackageDirNameFlagName) + } + + patterns := splitAndTrim(packageDirNamePatterns, ",") + for _, patternString := range patterns { + pattern, err := glob.Compile(patternString) + if err != nil { + return fmt.Errorf("invalid package dir name pattern: %s: %w", patternString, err) + } + f.patterns = append(f.patterns, pattern) + } + + if len(f.patterns) > 0 { + f.isApplied = true + } + + return nil +} + +func (f *PackageDirNameFlag) Validate() error { + return nil +} + +func (f *PackageDirNameFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + for _, pattern := range f.patterns { + if pattern.Match(dirName) { + return true + } + } + return false +} + +func (f *PackageDirNameFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initPackageDirNameFlag() *PackageDirNameFlag { + return &PackageDirNameFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterPackageDirNameFlagName, + description: cobraext.FilterPackageDirNameFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/packagename.go b/internal/filter/packagename.go new file mode 100644 index 0000000000..63a415f298 --- /dev/null +++ b/internal/filter/packagename.go @@ -0,0 +1,77 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + + "github.com/gobwas/glob" + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type PackageNameFlag struct { + FilterFlagBase + + patterns []glob.Glob +} + +func (f *PackageNameFlag) Parse(cmd *cobra.Command) error { + packageNamePatterns, err := cmd.Flags().GetString(cobraext.FilterPackagesFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterPackagesFlagName) + } + + patterns := splitAndTrim(packageNamePatterns, ",") + for _, patternString := range patterns { + pattern, err := glob.Compile(patternString) + if err != nil { + return fmt.Errorf("invalid package name pattern: %s: %w", patternString, err) + } + f.patterns = append(f.patterns, pattern) + } + + if len(f.patterns) > 0 { + f.isApplied = true + } + + return nil +} + +func (f *PackageNameFlag) Validate() error { + return nil +} + +func (f *PackageNameFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + for _, pattern := range f.patterns { + if pattern.Match(manifest.Name) { + return true + } + } + return false +} + +func (f *PackageNameFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initPackageNameFlag() *PackageNameFlag { + return &PackageNameFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterPackagesFlagName, + description: cobraext.FilterPackagesFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/packagetype.go b/internal/filter/packagetype.go new file mode 100644 index 0000000000..d1945a4b2f --- /dev/null +++ b/internal/filter/packagetype.go @@ -0,0 +1,61 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type PackageTypeFlag struct { + FilterFlagBase + + // flag specific fields + values []string +} + +func (f *PackageTypeFlag) Parse(cmd *cobra.Command) error { + packageTypes, err := cmd.Flags().GetString(cobraext.FilterPackageTypeFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterPackageTypeFlagName) + } + if packageTypes == "" { + return nil + } + f.values = splitAndTrim(packageTypes, ",") + f.isApplied = true + return nil +} + +func (f *PackageTypeFlag) Validate() error { + return nil +} + +func (f *PackageTypeFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + return hasAnyMatch(f.values, []string{manifest.Type}) +} + +func (f *PackageTypeFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initPackageTypeFlag() *PackageTypeFlag { + return &PackageTypeFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterPackageTypeFlagName, + description: cobraext.FilterPackageTypeFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/registry.go b/internal/filter/registry.go new file mode 100644 index 0000000000..0066a413f3 --- /dev/null +++ b/internal/filter/registry.go @@ -0,0 +1,109 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/multierror" + "github.com/elastic/elastic-package/internal/packages" +) + +var registry = []Filter{ + initCategoryFlag(), + initCodeOwnerFlag(), + initInputFlag(), + initPackageDirNameFlag(), + initPackageNameFlag(), + initPackageTypeFlag(), + initSpecVersionFlag(), +} + +// SetFilterFlags registers all filter flags with the given command. +func SetFilterFlags(cmd *cobra.Command) { + cmd.Flags().IntP(cobraext.FilterDepthFlagName, cobraext.FilterDepthFlagShorthand, cobraext.FilterDepthFlagDefault, cobraext.FilterDepthFlagDescription) + cmd.Flags().StringP(cobraext.FilterExcludeDirFlagName, "", "", cobraext.FilterExcludeDirFlagDescription) + + for _, filterFlag := range registry { + filterFlag.Register(cmd) + } +} + +// FilterRegistry manages a collection of filters for package filtering. +type FilterRegistry struct { + filters []Filter + depth int + excludeDirs string +} + +// NewFilterRegistry creates a new FilterRegistry instance. +func NewFilterRegistry(depth int, excludeDirs string) *FilterRegistry { + return &FilterRegistry{ + filters: []Filter{}, + depth: depth, + excludeDirs: excludeDirs, + } +} + +func (r *FilterRegistry) Parse(cmd *cobra.Command) error { + errs := multierror.Error{} + for _, filter := range registry { + if err := filter.Parse(cmd); err != nil { + errs = append(errs, err) + } + + if filter.IsApplied() { + r.filters = append(r.filters, filter) + } + } + + if errs.Error() != "" { + return fmt.Errorf("error parsing filter options: %s", errs.Error()) + } + + return nil +} + +func (r *FilterRegistry) Validate() error { + for _, filter := range r.filters { + if err := filter.Validate(); err != nil { + return err + } + } + return nil +} + +func (r *FilterRegistry) Execute() (filtered []packages.PackageDirNameAndManifest, errors multierror.Error) { + currentDir, err := os.Getwd() + if err != nil { + return nil, multierror.Error{fmt.Errorf("getting current directory failed: %w", err)} + } + + pkgs, err := packages.ReadAllPackageManifestsFromRepo(currentDir, r.depth, r.excludeDirs) + if err != nil { + return nil, multierror.Error{err} + } + + filtered = pkgs + for _, filter := range r.filters { + logger.Infof("Applying for %d packages", len(filtered)) + filtered, err = filter.ApplyTo(filtered) + if err != nil { + errors = append(errors, err) + } + + if len(filtered) == 0 { + break + } + } + + logger.Infof("Found %d matching package(s)\n", len(filtered)) + return filtered, errors +} diff --git a/internal/filter/specversion.go b/internal/filter/specversion.go new file mode 100644 index 0000000000..c88cc70e6f --- /dev/null +++ b/internal/filter/specversion.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + + "github.com/Masterminds/semver/v3" + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/cobraext" + "github.com/elastic/elastic-package/internal/packages" +) + +type SpecVersionFlag struct { + FilterFlagBase + + // package spec version constraint + constraints *semver.Constraints +} + +func (f *SpecVersionFlag) Parse(cmd *cobra.Command) error { + specVersion, err := cmd.Flags().GetString(cobraext.FilterSpecVersionFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.FilterSpecVersionFlagName) + } + if specVersion == "" { + return nil + } + + f.constraints, err = semver.NewConstraint(specVersion) + if err != nil { + return fmt.Errorf("invalid spec version: %s: %w", specVersion, err) + } + + f.isApplied = true + return nil +} + +func (f *SpecVersionFlag) Validate() error { + // no validation needed for this flag + // checks are done in Parse method + return nil +} + +func (f *SpecVersionFlag) Matches(dirName string, manifest *packages.PackageManifest) bool { + pkgVersion, err := semver.NewVersion(manifest.SpecVersion) + if err != nil { + return false + } + return f.constraints.Check(pkgVersion) +} + +func (f *SpecVersionFlag) ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) { + filtered := make([]packages.PackageDirNameAndManifest, 0, len(pkgs)) + for _, pkg := range pkgs { + if f.Matches(pkg.DirName, pkg.Manifest) { + filtered = append(filtered, pkg) + } + } + return filtered, nil +} + +func initSpecVersionFlag() *SpecVersionFlag { + return &SpecVersionFlag{ + FilterFlagBase: FilterFlagBase{ + name: cobraext.FilterSpecVersionFlagName, + description: cobraext.FilterSpecVersionFlagDescription, + shorthand: "", + defaultValue: "", + }, + } +} diff --git a/internal/filter/type.go b/internal/filter/type.go new file mode 100644 index 0000000000..90bd2ea391 --- /dev/null +++ b/internal/filter/type.go @@ -0,0 +1,55 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/elastic/elastic-package/internal/packages" +) + +// FilterFlag defines the basic interface for filter flags. +type FilterFlag interface { + String() string + Register(cmd *cobra.Command) + IsApplied() bool +} + +// Filter extends FilterFlag with filtering capabilities. +// It defines the interface for filtering packages based on specific criteria. +type Filter interface { + FilterFlag + + Parse(cmd *cobra.Command) error + Validate() error + ApplyTo(pkgs []packages.PackageDirNameAndManifest) ([]packages.PackageDirNameAndManifest, error) + // Matches checks if a package matches the filter criteria. + // dirName is the directory name of the package in package root. + Matches(dirName string, manifest *packages.PackageManifest) bool +} + +// FilterFlagBase provides common functionality for filter flags. +type FilterFlagBase struct { + name string + description string + shorthand string + defaultValue string + + isApplied bool +} + +func (f *FilterFlagBase) String() string { + return fmt.Sprintf("name=%s defaultValue=%s applied=%v", f.name, f.defaultValue, f.isApplied) +} + +func (f *FilterFlagBase) Register(cmd *cobra.Command) { + cmd.Flags().StringP(f.name, f.shorthand, f.defaultValue, f.description) +} + +func (f *FilterFlagBase) IsApplied() bool { + return f.isApplied +} diff --git a/internal/filter/utils.go b/internal/filter/utils.go new file mode 100644 index 0000000000..d1b416a019 --- /dev/null +++ b/internal/filter/utils.go @@ -0,0 +1,64 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package filter + +import ( + "slices" + "strings" + + "github.com/elastic/elastic-package/internal/packages" +) + +// splitAndTrim splits a string by delimiter and trims whitespace from each element +func splitAndTrim(s, delimiter string) []string { + if s == "" { + return nil + } + parts := strings.Split(s, delimiter) + result := make([]string, 0, len(parts)) + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if trimmed != "" { + result = append(result, trimmed) + } + } + return result +} + +// hasAnyMatch checks if any item in the items slice exists in the filters slice +func hasAnyMatch(filters []string, items []string) bool { + if len(filters) == 0 { + return true + } + + for _, item := range items { + if slices.Contains(filters, item) { + return true + } + } + + return false +} + +// extractInputs extracts all input types from package policy templates +func extractInputs(manifest *packages.PackageManifest) []string { + uniqueInputs := make(map[string]struct{}) + for _, policyTemplate := range manifest.PolicyTemplates { + if policyTemplate.Input != "" { + uniqueInputs[policyTemplate.Input] = struct{}{} + } + + for _, input := range policyTemplate.Inputs { + uniqueInputs[input.Type] = struct{}{} + } + } + + inputs := make([]string, 0, len(uniqueInputs)) + for input := range uniqueInputs { + inputs = append(inputs, input) + } + + return inputs +} diff --git a/internal/packages/packages.go b/internal/packages/packages.go index 4e55f9dca7..42e6b48d90 100644 --- a/internal/packages/packages.go +++ b/internal/packages/packages.go @@ -201,6 +201,12 @@ type PackageManifest struct { Elasticsearch *Elasticsearch `config:"elasticsearch" json:"elasticsearch" yaml:"elasticsearch"` } +type PackageDirNameAndManifest struct { + DirName string + Path string + Manifest *PackageManifest +} + type ManifestIndexTemplate struct { IngestPipeline *ManifestIngestPipeline `config:"ingest_pipeline" json:"ingest_pipeline" yaml:"ingest_pipeline"` Mappings *ManifestMappings `config:"mappings" json:"mappings" yaml:"mappings"` @@ -419,6 +425,87 @@ func ReadPackageManifest(path string) (*PackageManifest, error) { return &m, nil } +// ReadAllPackageManifestsFromRepo reads all the package manifests in the given directory. +// It recursively searches for manifest.yml files up to the specified depth. +// - depth: maximum depth to search (1 = current dir + immediate sub dirs) +// - excludeDirs: comma-separated list of directory names to exclude (always excludes .git) +func ReadAllPackageManifestsFromRepo(searchRoot string, depth int, excludeDirs string) ([]PackageDirNameAndManifest, error) { + // Parse exclude directories + excludeMap := map[string]bool{ + ".git": true, // Always exclude .git + } + if excludeDirs != "" { + for dir := range strings.SplitSeq(excludeDirs, ",") { + excludeMap[strings.TrimSpace(dir)] = true + } + } + + var packages []PackageDirNameAndManifest + searchRootDepth := strings.Count(searchRoot, string(filepath.Separator)) + + err := filepath.WalkDir(searchRoot, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // Calculate current depth relative to search root + currentDepth := strings.Count(path, string(filepath.Separator)) - searchRootDepth + + // If it's a directory, check if we should skip it + if d.IsDir() { + dirName := d.Name() + + // Skip excluded directories + if excludeMap[dirName] { + return filepath.SkipDir + } + + // Skip if we've exceeded the depth limit (but allow processing the current level) + if currentDepth > depth { + return filepath.SkipDir + } + + return nil + } + + // Check if this is a manifest file + if d.Name() != PackageManifestFile { + return nil + } + + // Validate it's a package manifest + ok, err := isPackageManifest(path) + if err != nil { + // Log the error but continue searching + return nil + } + if !ok { + return nil + } + + // Extract directory name (just the package directory name, not the full path) + dirName := filepath.Base(filepath.Dir(path)) + manifest, err := ReadPackageManifest(path) + if err != nil { + return fmt.Errorf("failed to read package manifest (path: %s): %w", path, err) + } + + packages = append(packages, PackageDirNameAndManifest{ + DirName: dirName, + Manifest: manifest, + Path: filepath.Dir(path), + }) + + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed walking directory tree: %w", err) + } + + return packages, nil +} + // ReadTransformDefinitionFile reads and parses the transform definition (elasticsearch/transform//transform.yml) // file for the given transform. It also applies templating to the file, allowing to set the final ingest pipeline name // by adding the package version defined in the package manifest.