Add file-based authentication support for CEL and HTTP JSON inputs

changelog/fragments/1762465899-file-auth-httpjson-cel.yaml

@@ -0,0 +1,9 @@
+kind: enhancement
+summary: Add file-based auth provider for CEL and HTTP JSON inputs.
+description: |
+  The CEL and HTTP JSON inputs now support reading authentication tokens from
+  files, enabling integration with various secret providers like Vault,
+  Kubernetes secret projections, etc. Tokens are automatically refreshed based on
+  a configurable interval without requiring restarts.
+component: filebeat
+issue: https://github.com/elastic/beats/issues/47506

docs/reference/filebeat/filebeat-input-cel.md

@@ -21,6 +21,7 @@ This input supports:
 
     * Basic
     * Digest
+    * File
     * OAuth2
 
 * Retrieval at a configurable interval
@@ -271,9 +272,10 @@ Additionally, it supports authentication via:
 * Basic Authentication
 * Digest Authentication {applies_to}`stack: ga 8.12.0`
 * OAuth2
+* file-based headers
 * token authentication {applies_to}`stack: ga 8.19.0, unavailable 9.0.0, ga 9.1.0`
 
-As described in Mito's [HTTP]({{mito_docs}}@{{mito_version}}/lib#HTTP) documentation, configuration for Basic Authentication or token authentication will only affect direct HEAD, GET and POST method calls, not explicity constructed requests run with `.do_request()`. Configuration for Digest Authentication or OAuth2 will be used for all requests made from CEL.
+As described in Mito's [HTTP]({{mito_docs}}@{{mito_version}}/lib#HTTP) documentation, configuration for Basic Authentication or token authentication will only affect direct HEAD, GET and POST method calls, not explicity constructed requests run with `.do_request()`. Configuration for Digest Authentication, file-based headers or OAuth2 will be used for all requests made from CEL.
 
 Example configurations with authentication:
 
@@ -317,6 +319,16 @@ filebeat.inputs:
   resource.url: http://localhost
 ```
 
+```yaml
+filebeat.inputs:
+- type: cel
+  auth.file:
+    path: /etc/elastic/token
+    prefix: "Bearer "
+    refresh_interval: 10m
+  resource.url: http://localhost
+```
+
 ```yaml
 filebeat.inputs:
 - type: cel
@@ -557,6 +569,35 @@ stack: ga 8.12.0
 When set to `true`, Digest Authentication challenges are not reused.
 
 
+### `auth.file.enabled` [_auth_file_enabled]
+
+When set to `false`, disables the file auth configuration. Default: `true`.
+
+::::{note}
+File auth settings are disabled if either `enabled` is set to `false` or the `auth.file` section is missing.
+::::
+
+
+### `auth.file.path` [_auth_file_path]
+
+The path to the file containing the authentication value. The file contents are trimmed before use. This field is required when file auth is enabled.
+
+
+### `auth.file.header` [_auth_file_header]
+
+The request header that receives the value loaded from `path`. Defaults to `Authorization` when omitted or empty.
+
+
+### `auth.file.prefix` [_auth_file_prefix]
+
+An optional prefix that is prepended to the trimmed value from `path` before it is set on the request header. This is commonly used for tokens that require a leading value such as `Bearer `.
+
+
+### `auth.file.refresh_interval` [_auth_file_refresh_interval]
+
+How frequently Filebeat rereads the file defined by `path` to pick up changes. Defaults to `1m`. The value must be greater than zero when set.
+
+
 ### `auth.oauth2.enabled` [_auth_oauth2_enabled]
 
 When set to `false`, disables the oauth2 configuration. Default: `true`.

docs/reference/filebeat/filebeat-input-httpjson.md

@@ -17,8 +17,9 @@ This input supports:
 
 * Auth
 
-    * Basic
-    * OAuth2
+  * Basic
+  * File
+  * OAuth2
 
 * Retrieval at a configurable interval
 * Pagination
@@ -61,7 +62,7 @@ filebeat.inputs:
         value: 5m
 ```
 
-Additionally, it supports authentication via Basic auth, HTTP Headers or oauth2.
+Additionally, it supports authentication via Basic auth, file-based headers (`auth.file`), HTTP headers, or oauth2.
 
 Example configurations with authentication:
 
@@ -97,6 +98,16 @@ filebeat.inputs:
   request.url: http://localhost
 ```
 
+```yaml
+filebeat.inputs:
+- type: httpjson
+  auth.file:
+    path: /etc/elastic/token
+    prefix: "Bearer "
+    refresh_interval: 10m
+  request.url: http://localhost
+```
+
 ## Input state [input-state]
 
 The `httpjson` input keeps a runtime state between requests. This state can be accessed by some configuration options and transforms.
@@ -261,6 +272,35 @@ The user to authenticate with.
 The password to use.
 
 
+### `auth.file.enabled` [_auth_file_enabled_2]
+
+When set to `false`, disables the file auth configuration. Default: `true`.
+
+::::{note}
+File auth settings are disabled if either `enabled` is set to `false` or the `auth.file` section is missing.
+::::
+
+
+### `auth.file.path` [_auth_file_path_2]
+
+The path to the file that contains the authentication value. The file contents are trimmed before use. This field is required when file auth is enabled.
+
+
+### `auth.file.header` [_auth_file_header_2]
+
+The request header that receives the value loaded from `path`. Defaults to `Authorization` when omitted or empty.
+
+
+### `auth.file.prefix` [_auth_file_prefix_2]
+
+An optional prefix that is prepended to the trimmed value from `path` before it is sent on the request header. This is commonly used for tokens that require a leading value such as `Bearer `.
+
+
+### `auth.file.refresh_interval` [_auth_file_refresh_interval_2]
+
+How frequently Filebeat rereads the file defined by `path` to pick up changes. Defaults to `1m`. The value must be greater than zero when set.
+
+
 ### `auth.oauth2.enabled` [_auth_oauth2_enabled_2]
 
 When set to `false`, disables the oauth2 configuration. Default: `true`.

x-pack/filebeat/input/cel/config_auth.go

@@ -14,6 +14,7 @@ import (
 	"net/url"
 	"os"
 	"strings"
+	"time"
 
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
@@ -28,10 +29,16 @@ type authConfig struct {
 	Basic  *basicAuthConfig       `config:"basic"`
 	Token  *tokenAuthConfig       `config:"token"`
 	Digest *digestAuthConfig      `config:"digest"`
+	File   *fileAuthConfig        `config:"file"`
 	OAuth2 *oAuth2Config          `config:"oauth2"`
 	AWS    *aws.SignerInputConfig `config:"aws"`
 }
 
+const (
+	defaultFileAuthHeader          = "Authorization"
+	defaultFileAuthRefreshInterval = time.Minute
+)
+
 func (c authConfig) Validate() error {
 	var n int
 	if c.Basic.isEnabled() {
@@ -43,6 +50,9 @@ func (c authConfig) Validate() error {
 	if c.Digest.isEnabled() {
 		n++
 	}
+	if c.File.isEnabled() {
+		n++
+	}
 	if c.OAuth2.isEnabled() {
 		n++
 	}
@@ -128,6 +138,45 @@ func (d *digestAuthConfig) Validate() error {
 	return nil
 }
 
+type fileAuthConfig struct {
+	Enabled         *bool          `config:"enabled"`
+	Path            string         `config:"path"`
+	Header          string         `config:"header"`
+	Prefix          string         `config:"prefix"`
+	RefreshInterval *time.Duration `config:"refresh_interval"`
+}
+
+func (f *fileAuthConfig) isEnabled() bool {
+	return f != nil && (f.Enabled == nil || *f.Enabled)
+}
+
+func (f *fileAuthConfig) Validate() error {
+	if !f.isEnabled() {
+		return nil
+	}
+	if f.Path == "" {
+		return errors.New("path must be set")
+	}
+	if f.RefreshInterval != nil && (*f.RefreshInterval <= 0) {
+		return errors.New("refresh_interval must be greater than 0")
+	}
+	return nil
+}
+
+func (f *fileAuthConfig) headerName() string {
+	if f == nil || strings.TrimSpace(f.Header) == "" {
+		return defaultFileAuthHeader
+	}
+	return f.Header
+}
+
+func (f *fileAuthConfig) refreshInterval() time.Duration {
+	if f == nil || f.RefreshInterval == nil {
+		return defaultFileAuthRefreshInterval
+	}
+	return *f.RefreshInterval
+}
+
 // An oAuth2Provider represents a supported oauth provider.
 type oAuth2Provider string
 

x-pack/filebeat/input/cel/config_test.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
 	"time"
@@ -63,6 +64,7 @@ func TestIsEnabled(t *testing.T) {
 	}{
 		{name: "basic", auth: &basicAuthConfig{}},
 		{name: "digest", auth: &digestAuthConfig{}},
+		{name: "file", auth: &fileAuthConfig{}},
 		{name: "OAuth2", auth: &oAuth2Config{}},
 	} {
 		t.Run(test.name, func(t *testing.T) {
@@ -90,8 +92,101 @@ func TestIsEnabled(t *testing.T) {
 // take methods are for testing only.
 func (b *basicAuthConfig) take(on *bool)  { b.Enabled = on }
 func (d *digestAuthConfig) take(on *bool) { d.Enabled = on }
+func (f *fileAuthConfig) take(on *bool)   { f.Enabled = on }
 func (o *oAuth2Config) take(on *bool)     { o.Enabled = on }
 
+func TestFileAuthConfigValidate(t *testing.T) {
+	t.Run("requires path", func(t *testing.T) {
+		cfg := &fileAuthConfig{}
+		if err := cfg.Validate(); err == nil || err.Error() != "path must be set" {
+			t.Fatalf("expected path requirement error, got: %v", err)
+		}
+	})
+
+	t.Run("requires positive refresh interval", func(t *testing.T) {
+		path := filepath.Join(t.TempDir(), "token")
+		zero := time.Duration(0)
+		cfg := &fileAuthConfig{Path: path, RefreshInterval: &zero}
+		if err := cfg.Validate(); err == nil || err.Error() != "refresh_interval must be greater than 0" {
+			t.Fatalf("expected refresh interval error, got: %v", err)
+		}
+	})
+
+	t.Run("valid configuration", func(t *testing.T) {
+		path := filepath.Join(t.TempDir(), "token")
+		refresh := time.Second
+		cfg := &fileAuthConfig{Path: path, RefreshInterval: &refresh}
+		if err := cfg.Validate(); err != nil {
+			t.Fatalf("unexpected validation error: %v", err)
+		}
+	})
+}
+
+func TestFileAuthConfigDefaults(t *testing.T) {
+	cfg := &fileAuthConfig{}
+	if got := cfg.headerName(); got != defaultFileAuthHeader {
+		t.Fatalf("unexpected default header: got %q want %q", got, defaultFileAuthHeader)
+	}
+	if got := cfg.refreshInterval(); got != defaultFileAuthRefreshInterval {
+		t.Fatalf("unexpected default refresh interval: got %v want %v", got, defaultFileAuthRefreshInterval)
+	}
+
+	header := "X-Api-Key"
+	cfg.Header = header
+	if got := cfg.headerName(); got != header {
+		t.Fatalf("unexpected header override: got %q want %q", got, header)
+	}
+
+	refresh := 42 * time.Second
+	cfg.RefreshInterval = &refresh
+	if got := cfg.refreshInterval(); got != refresh {
+		t.Fatalf("unexpected refresh interval override: got %v want %v", got, refresh)
+	}
+}
+
+func TestConfigFileAuthMutualExclusion(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "secret")
+	if err := os.WriteFile(path, []byte("secret"), 0o600); err != nil {
+		t.Fatalf("failed to write secret file: %v", err)
+	}
+
+	cfg := conf.MustNewConfigFrom(map[string]interface{}{
+		"resource.url":        "localhost",
+		"auth.file.path":      path,
+		"auth.basic.user":     "user",
+		"auth.basic.password": "pass",
+	})
+	conf := defaultConfig()
+	conf.Program = "{}"
+	conf.Redact = &redact{}
+	err := cfg.Unpack(&conf)
+	wantErr := errors.New("only one kind of auth can be enabled accessing 'auth'")
+	if fmt.Sprint(err) != fmt.Sprint(wantErr) {
+		t.Fatalf("unexpected error: got %v want %v", err, wantErr)
+	}
+}
+
+func TestConfigFileAuthDisabledAllowsOther(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "secret")
+	if err := os.WriteFile(path, []byte("secret"), 0o600); err != nil {
+		t.Fatalf("failed to write secret file: %v", err)
+	}
+
+	cfg := conf.MustNewConfigFrom(map[string]interface{}{
+		"resource.url":        "localhost",
+		"auth.file.enabled":   false,
+		"auth.file.path":      path,
+		"auth.basic.user":     "user",
+		"auth.basic.password": "pass",
+	})
+	conf := defaultConfig()
+	conf.Program = "{}"
+	conf.Redact = &redact{}
+	if err := cfg.Unpack(&conf); err != nil {
+		t.Fatalf("unexpected error unpacking config: %v", err)
+	}
+}
+
 func TestOAuth2GetTokenURL(t *testing.T) {
 	const host = "http://localhost"
 	for _, test := range []struct {