diff --git a/changelog/fragments/1764010000-translate-ldap-guid-inference.yaml b/changelog/fragments/1764010000-translate-ldap-guid-inference.yaml new file mode 100644 index 000000000000..28b39d0dd6e1 --- /dev/null +++ b/changelog/fragments/1764010000-translate-ldap-guid-inference.yaml @@ -0,0 +1,45 @@ +# REQUIRED +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: enhancement + +# REQUIRED for all kinds +# Change summary; a 80ish characters long description of the change. +summary: Add GUID translation, base DN inference, and SSPI authentication to LDAP processor. + +# REQUIRED for breaking-change, deprecation, known-issue +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# description: + +# REQUIRED for breaking-change, deprecation, known-issue +# impact: + +# REQUIRED for breaking-change, deprecation, known-issue +# action: + +# REQUIRED for all kinds +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: libbeat + +# AUTOMATED +# OPTIONAL to manually add other PR URLs +# PR URL: A link the PR that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +pr: https://github.com/elastic/beats/pull/47827 + +# AUTOMATED +# OPTIONAL to manually add other issue URLs +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +# issue: https://github.com/owner/repo/1234 diff --git a/docs/reference/auditbeat/processor-translate-guid.md b/docs/reference/auditbeat/processor-translate-guid.md index d5a4514f4e86..6db349507687 100644 --- a/docs/reference/auditbeat/processor-translate-guid.md +++ b/docs/reference/auditbeat/processor-translate-guid.md @@ -6,27 +6,27 @@ applies_to: stack: ga --- -# Translate GUID [processor-translate-guid] +# Translate LDAP Attribute [processor-translate-ldap-attribute] +The `translate_ldap_attribute` processor translates LDAP attributes into friendlier values. The typical use case is converting an Active Directory Global Unique Identifier (GUID) into a human-readable name (for example the object's `cn`). -The `translate_ldap_attribute` processor translates an LDAP attributes between eachother. It is typically used to translate AD Global Unique Identifiers (GUID) into their common names. +Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object's name and these values sometimes appear in logs. -Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object’s name and these values sometimes appear in logs. - -If the search attribute is invalid (malformed) or does not map to any object on the domain then this will result in the processor returning an error unless `ignore_failure` is set. +If the search attribute is invalid (malformed) or does not map to any object on the domain the processor returns an error unless `ignore_failure` is set. The result of this operation is an array of values, given that a single attribute can hold multiple values. -Note: the search attribute is expected to map to a single object. If it doesn’t, no error will be returned, but only results of the first entry will be added to the event. +Note: the search attribute is expected to map to a single object. If multiple entries match, only the first entry's mapped attribute values are returned. ```yaml processors: - translate_ldap_attribute: field: winlog.event_data.ObjectGuid - ldap_address: "ldap://" - ldap_base_dn: "dc=example,dc=com" ignore_missing: true ignore_failure: true + # ldap_domain: "example.com" # Optional - override auto-discovered domain for DNS SRV queries + # ldap_address: "ldap://ds.example.com:389" # Optional - resolve via DNS SRV/LOGONSERVER when omitted + # ldap_base_dn: "dc=example,dc=com" # Optional - discovered via rootDSE or inferred from server domain ``` The `translate_ldap_attribute` processor has the following configuration settings: @@ -35,18 +35,38 @@ The `translate_ldap_attribute` processor has the following configuration setting | --- | --- | --- | --- | | `field` | yes | | Source field containing a GUID. | | `target_field` | no | | Target field for the mapped attribute value. If not set it will be replaced in place. | -| `ldap_address` | yes | | LDAP server address. eg: `ldap://ds.example.com:389` | -| `ldap_base_dn` | yes | | LDAP base DN. eg: `dc=example,dc=com` | -| `ldap_bind_user` | no | | LDAP user. | -| `ldap_bind_password` | no | | LDAP password. | +| `ldap_domain` | no | | DNS domain name for DNS SRV lookups (e.g., `example.com`). When omitted, the domain is auto-discovered from the `USERDNSDOMAIN` environment variable (Windows) or the hostname's domain suffix. This setting is used only when `ldap_address` is not provided. | +| `ldap_address` | no | | LDAP server address (eg: `ldap://ds.example.com:389`). If not provided, auto-discovery will be attempted via DNS SRV records and, on Windows, the LOGONSERVER environment variable. | +| `ldap_base_dn` | no | | LDAP base DN (eg: `dc=example,dc=com`). If not provided, auto-discovery will be attempted via rootDSE query or inferred from the server domain. | +| `ldap_bind_user` | no | | LDAP user. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | +| `ldap_bind_password` | no | | LDAP password. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | | `ldap_search_attribute` | yes | `objectGUID` | LDAP attribute to search by. | | `ldap_mapped_attribute` | yes | `cn` | LDAP attribute to map to. | | `ldap_search_time_limit` | no | 30 | LDAP search time limit in seconds. | -| `ldap_ssl`\* | no | 30 | LDAP TLS/SSL connection settings. | +| `ldap_ssl`* | no | | LDAP TLS/SSL connection settings. See [SSL](/reference/auditbeat/configuration-ssl.md). | +| `ad_guid_translation` | no | `auto` | Controls GUID binary conversion for Active Directory attributes. `auto` (default) converts when the LDAP search attribute equals `objectGUID` (case-insensitive). Use `always` to force conversion or `never` to disable it. | | `ignore_missing` | no | false | Ignore errors when the source field is missing. | | `ignore_failure` | no | false | Ignore all errors produced by the processor. | -\* Also see [SSL](/reference/auditbeat/configuration-ssl.md) for a full description of the `ldap_ssl` options. +## Server auto-discovery + +When `ldap_address` is omitted the processor attempts to discover controllers in the following order: + +1. DNS SRV lookups for `_ldaps._tcp` (preferred) and `_ldap._tcp` using the system's native DNS resolver. The processor queries multiple patterns: + - `_ldap._tcp.dc._msdcs.` (Active Directory domain controllers) + - `_ldap._tcp.` (standard domain lookup) + - If no domain is available, bare queries like `_ldap._tcp` which automatically use the system's DNS search suffix configuration (works on Windows, Linux, and macOS) + + The domain used for DNS SRV lookups is determined from: + - The `ldap_domain` configuration option (highest priority) + - Operating-system domain metadata (for example session environment values or Windows domain-join settings) + - The host's fully qualified name when available + - Reverse DNS lookup of the local machine as a last resort + + **Note:** The processor uses Go's standard library `net.LookupSRV()` which leverages the operating system's native DNS resolver. On Windows, this automatically reads DNS servers and search suffixes from the Windows registry, making autodiscovery work correctly even when running as a service without environment variables. SRV records are automatically ordered by priority with weight-based randomization per RFC 2782 to distribute load across available servers. +2. On Windows, the `LOGONSERVER` environment variable. The processor keeps the hostname for TLS validation and may also try the resolved IP as a fallback. + +Each candidate server is tried sequentially until one responds. Likewise, if `ldap_base_dn` is not supplied the client queries the server's rootDSE for `defaultNamingContext`/`namingContexts`, and if that fails, infers the DN from the server's domain name (for example `dc=example,dc=com`). If the searches are slow or you expect a high amount of different key attributes to be found, consider using a cache processor to speed processing: @@ -78,4 +98,3 @@ processors: key_field: winlog.event_data.ObjectGuid value_field: winlog.common_name ``` - diff --git a/docs/reference/filebeat/processor-translate-guid.md b/docs/reference/filebeat/processor-translate-guid.md index 89af9d4c963b..32287bab46a9 100644 --- a/docs/reference/filebeat/processor-translate-guid.md +++ b/docs/reference/filebeat/processor-translate-guid.md @@ -6,27 +6,27 @@ applies_to: stack: ga --- -# Translate GUID [processor-translate-guid] +# Translate LDAP Attribute [processor-translate-ldap-attribute] +The `translate_ldap_attribute` processor translates LDAP attributes into friendlier values. The typical use case is converting an Active Directory Global Unique Identifier (GUID) into a human-readable name (for example the object's `cn`). -The `translate_ldap_attribute` processor translates an LDAP attributes between eachother. It is typically used to translate AD Global Unique Identifiers (GUID) into their common names. +Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object's name and these values sometimes appear in logs. -Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object’s name and these values sometimes appear in logs. - -If the search attribute is invalid (malformed) or does not map to any object on the domain then this will result in the processor returning an error unless `ignore_failure` is set. +If the search attribute is invalid (malformed) or does not map to any object on the domain the processor returns an error unless `ignore_failure` is set. The result of this operation is an array of values, given that a single attribute can hold multiple values. -Note: the search attribute is expected to map to a single object. If it doesn’t, no error will be returned, but only results of the first entry will be added to the event. +Note: the search attribute is expected to map to a single object. If multiple entries match, only the first entry's mapped attribute values are returned. ```yaml processors: - translate_ldap_attribute: field: winlog.event_data.ObjectGuid - ldap_address: "ldap://" - ldap_base_dn: "dc=example,dc=com" ignore_missing: true ignore_failure: true + # ldap_domain: "example.com" # Optional - override auto-discovered domain for DNS SRV queries + # ldap_address: "ldap://ds.example.com:389" # Optional - resolve via DNS SRV/LOGONSERVER when omitted + # ldap_base_dn: "dc=example,dc=com" # Optional - discovered via rootDSE or inferred from server domain ``` The `translate_ldap_attribute` processor has the following configuration settings: @@ -35,18 +35,38 @@ The `translate_ldap_attribute` processor has the following configuration setting | --- | --- | --- | --- | | `field` | yes | | Source field containing a GUID. | | `target_field` | no | | Target field for the mapped attribute value. If not set it will be replaced in place. | -| `ldap_address` | yes | | LDAP server address. eg: `ldap://ds.example.com:389` | -| `ldap_base_dn` | yes | | LDAP base DN. eg: `dc=example,dc=com` | -| `ldap_bind_user` | no | | LDAP user. | -| `ldap_bind_password` | no | | LDAP password. | +| `ldap_domain` | no | | DNS domain name for DNS SRV lookups (e.g., `example.com`). When omitted, the domain is auto-discovered from the `USERDNSDOMAIN` environment variable (Windows) or the hostname's domain suffix. This setting is used only when `ldap_address` is not provided. | +| `ldap_address` | no | | LDAP server address (eg: `ldap://ds.example.com:389`). If not provided, auto-discovery will be attempted via DNS SRV records and, on Windows, the LOGONSERVER environment variable. | +| `ldap_base_dn` | no | | LDAP base DN (eg: `dc=example,dc=com`). If not provided, auto-discovery will be attempted via rootDSE query or inferred from the server domain. | +| `ldap_bind_user` | no | | LDAP user. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | +| `ldap_bind_password` | no | | LDAP password. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | | `ldap_search_attribute` | yes | `objectGUID` | LDAP attribute to search by. | | `ldap_mapped_attribute` | yes | `cn` | LDAP attribute to map to. | | `ldap_search_time_limit` | no | 30 | LDAP search time limit in seconds. | -| `ldap_ssl`\* | no | 30 | LDAP TLS/SSL connection settings. | +| `ldap_ssl`* | no | | LDAP TLS/SSL connection settings. See [SSL](/reference/filebeat/configuration-ssl.md). | +| `ad_guid_translation` | no | `auto` | Controls GUID binary conversion for Active Directory attributes. `auto` (default) converts when the LDAP search attribute equals `objectGUID` (case-insensitive). Use `always` to force conversion or `never` to disable it. | | `ignore_missing` | no | false | Ignore errors when the source field is missing. | | `ignore_failure` | no | false | Ignore all errors produced by the processor. | -\* Also see [SSL](/reference/filebeat/configuration-ssl.md) for a full description of the `ldap_ssl` options. +## Server auto-discovery + +When `ldap_address` is omitted the processor attempts to discover controllers in the following order: + +1. DNS SRV lookups for `_ldaps._tcp` (preferred) and `_ldap._tcp` using the system's native DNS resolver. The processor queries multiple patterns: + - `_ldap._tcp.dc._msdcs.` (Active Directory domain controllers) + - `_ldap._tcp.` (standard domain lookup) + - If no domain is available, bare queries like `_ldap._tcp` which automatically use the system's DNS search suffix configuration (works on Windows, Linux, and macOS) + + The domain used for DNS SRV lookups is determined from: + - The `ldap_domain` configuration option (highest priority) + - Operating-system domain metadata (for example session environment values or Windows domain-join settings) + - The host's fully qualified name when available + - Reverse DNS lookup of the local machine as a last resort + + **Note:** The processor uses Go's standard library `net.LookupSRV()` which leverages the operating system's native DNS resolver. On Windows, this automatically reads DNS servers and search suffixes from the Windows registry, making autodiscovery work correctly even when running as a service without environment variables. SRV records are automatically ordered by priority with weight-based randomization per RFC 2782 to distribute load across available servers. +2. On Windows, the `LOGONSERVER` environment variable. The processor keeps the hostname for TLS validation and may also try the resolved IP as a fallback. + +Each candidate server is tried sequentially until one responds. Likewise, if `ldap_base_dn` is not supplied the client queries the server's rootDSE for `defaultNamingContext`/`namingContexts`, and if that fails, infers the DN from the server's domain name (for example `dc=example,dc=com`). If the searches are slow or you expect a high amount of different key attributes to be found, consider using a cache processor to speed processing: @@ -78,4 +98,3 @@ processors: key_field: winlog.event_data.ObjectGuid value_field: winlog.common_name ``` - diff --git a/docs/reference/heartbeat/processor-translate-guid.md b/docs/reference/heartbeat/processor-translate-guid.md index 5984145c89f3..b45b77b69e0c 100644 --- a/docs/reference/heartbeat/processor-translate-guid.md +++ b/docs/reference/heartbeat/processor-translate-guid.md @@ -6,27 +6,27 @@ applies_to: stack: ga --- -# Translate GUID [processor-translate-guid] +# Translate LDAP Attribute [processor-translate-ldap-attribute] +The `translate_ldap_attribute` processor translates LDAP attributes into friendlier values. The typical use case is converting an Active Directory Global Unique Identifier (GUID) into a human-readable name (for example the object's `cn`). -The `translate_ldap_attribute` processor translates an LDAP attributes between eachother. It is typically used to translate AD Global Unique Identifiers (GUID) into their common names. +Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object's name and these values sometimes appear in logs. -Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object’s name and these values sometimes appear in logs. - -If the search attribute is invalid (malformed) or does not map to any object on the domain then this will result in the processor returning an error unless `ignore_failure` is set. +If the search attribute is invalid (malformed) or does not map to any object on the domain the processor returns an error unless `ignore_failure` is set. The result of this operation is an array of values, given that a single attribute can hold multiple values. -Note: the search attribute is expected to map to a single object. If it doesn’t, no error will be returned, but only results of the first entry will be added to the event. +Note: the search attribute is expected to map to a single object. If multiple entries match, only the first entry's mapped attribute values are returned. ```yaml processors: - translate_ldap_attribute: field: winlog.event_data.ObjectGuid - ldap_address: "ldap://" - ldap_base_dn: "dc=example,dc=com" ignore_missing: true ignore_failure: true + # ldap_domain: "example.com" # Optional - override auto-discovered domain for DNS SRV queries + # ldap_address: "ldap://ds.example.com:389" # Optional - resolve via DNS SRV/LOGONSERVER when omitted + # ldap_base_dn: "dc=example,dc=com" # Optional - discovered via rootDSE or inferred from server domain ``` The `translate_ldap_attribute` processor has the following configuration settings: @@ -35,18 +35,38 @@ The `translate_ldap_attribute` processor has the following configuration setting | --- | --- | --- | --- | | `field` | yes | | Source field containing a GUID. | | `target_field` | no | | Target field for the mapped attribute value. If not set it will be replaced in place. | -| `ldap_address` | yes | | LDAP server address. eg: `ldap://ds.example.com:389` | -| `ldap_base_dn` | yes | | LDAP base DN. eg: `dc=example,dc=com` | -| `ldap_bind_user` | no | | LDAP user. | -| `ldap_bind_password` | no | | LDAP password. | +| `ldap_domain` | no | | DNS domain name for DNS SRV lookups (e.g., `example.com`). When omitted, the domain is auto-discovered from the `USERDNSDOMAIN` environment variable (Windows) or the hostname's domain suffix. This setting is used only when `ldap_address` is not provided. | +| `ldap_address` | no | | LDAP server address (eg: `ldap://ds.example.com:389`). If not provided, auto-discovery will be attempted via DNS SRV records and, on Windows, the LOGONSERVER environment variable. | +| `ldap_base_dn` | no | | LDAP base DN (eg: `dc=example,dc=com`). If not provided, auto-discovery will be attempted via rootDSE query or inferred from the server domain. | +| `ldap_bind_user` | no | | LDAP user. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | +| `ldap_bind_password` | no | | LDAP password. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | | `ldap_search_attribute` | yes | `objectGUID` | LDAP attribute to search by. | | `ldap_mapped_attribute` | yes | `cn` | LDAP attribute to map to. | | `ldap_search_time_limit` | no | 30 | LDAP search time limit in seconds. | -| `ldap_ssl`\* | no | 30 | LDAP TLS/SSL connection settings. | +| `ldap_ssl`* | no | | LDAP TLS/SSL connection settings. See [SSL](/reference/heartbeat/configuration-ssl.md). | +| `ad_guid_translation` | no | `auto` | Controls GUID binary conversion for Active Directory attributes. `auto` (default) converts when the LDAP search attribute equals `objectGUID` (case-insensitive). Use `always` to force conversion or `never` to disable it. | | `ignore_missing` | no | false | Ignore errors when the source field is missing. | | `ignore_failure` | no | false | Ignore all errors produced by the processor. | -\* Also see [SSL](/reference/heartbeat/configuration-ssl.md) for a full description of the `ldap_ssl` options. +## Server auto-discovery + +When `ldap_address` is omitted the processor attempts to discover controllers in the following order: + +1. DNS SRV lookups for `_ldaps._tcp` (preferred) and `_ldap._tcp` using the system's native DNS resolver. The processor queries multiple patterns: + - `_ldap._tcp.dc._msdcs.` (Active Directory domain controllers) + - `_ldap._tcp.` (standard domain lookup) + - If no domain is available, bare queries like `_ldap._tcp` which automatically use the system's DNS search suffix configuration (works on Windows, Linux, and macOS) + + The domain used for DNS SRV lookups is determined from: + - The `ldap_domain` configuration option (highest priority) + - Operating-system domain metadata (for example session environment values or Windows domain-join settings) + - The host's fully qualified name when available + - Reverse DNS lookup of the local machine as a last resort + + **Note:** The processor uses Go's standard library `net.LookupSRV()` which leverages the operating system's native DNS resolver. On Windows, this automatically reads DNS servers and search suffixes from the Windows registry, making autodiscovery work correctly even when running as a service without environment variables. SRV records are automatically ordered by priority with weight-based randomization per RFC 2782 to distribute load across available servers. +2. On Windows, the `LOGONSERVER` environment variable. The processor keeps the hostname for TLS validation and may also try the resolved IP as a fallback. + +Each candidate server is tried sequentially until one responds. Likewise, if `ldap_base_dn` is not supplied the client queries the server's rootDSE for `defaultNamingContext`/`namingContexts`, and if that fails, infers the DN from the server's domain name (for example `dc=example,dc=com`). If the searches are slow or you expect a high amount of different key attributes to be found, consider using a cache processor to speed processing: @@ -78,4 +98,3 @@ processors: key_field: winlog.event_data.ObjectGuid value_field: winlog.common_name ``` - diff --git a/docs/reference/metricbeat/processor-translate-guid.md b/docs/reference/metricbeat/processor-translate-guid.md index 6eef946fe5d2..a74e5566285c 100644 --- a/docs/reference/metricbeat/processor-translate-guid.md +++ b/docs/reference/metricbeat/processor-translate-guid.md @@ -6,27 +6,27 @@ applies_to: stack: ga --- -# Translate GUID [processor-translate-guid] +# Translate LDAP Attribute [processor-translate-ldap-attribute] +The `translate_ldap_attribute` processor translates LDAP attributes into friendlier values. The typical use case is converting an Active Directory Global Unique Identifier (GUID) into a human-readable name (for example the object's `cn`). -The `translate_ldap_attribute` processor translates an LDAP attributes between eachother. It is typically used to translate AD Global Unique Identifiers (GUID) into their common names. +Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object's name and these values sometimes appear in logs. -Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object’s name and these values sometimes appear in logs. - -If the search attribute is invalid (malformed) or does not map to any object on the domain then this will result in the processor returning an error unless `ignore_failure` is set. +If the search attribute is invalid (malformed) or does not map to any object on the domain the processor returns an error unless `ignore_failure` is set. The result of this operation is an array of values, given that a single attribute can hold multiple values. -Note: the search attribute is expected to map to a single object. If it doesn’t, no error will be returned, but only results of the first entry will be added to the event. +Note: the search attribute is expected to map to a single object. If multiple entries match, only the first entry's mapped attribute values are returned. ```yaml processors: - translate_ldap_attribute: field: winlog.event_data.ObjectGuid - ldap_address: "ldap://" - ldap_base_dn: "dc=example,dc=com" ignore_missing: true ignore_failure: true + # ldap_domain: "example.com" # Optional - override auto-discovered domain for DNS SRV queries + # ldap_address: "ldap://ds.example.com:389" # Optional - resolve via DNS SRV/LOGONSERVER when omitted + # ldap_base_dn: "dc=example,dc=com" # Optional - discovered via rootDSE or inferred from server domain ``` The `translate_ldap_attribute` processor has the following configuration settings: @@ -35,18 +35,38 @@ The `translate_ldap_attribute` processor has the following configuration setting | --- | --- | --- | --- | | `field` | yes | | Source field containing a GUID. | | `target_field` | no | | Target field for the mapped attribute value. If not set it will be replaced in place. | -| `ldap_address` | yes | | LDAP server address. eg: `ldap://ds.example.com:389` | -| `ldap_base_dn` | yes | | LDAP base DN. eg: `dc=example,dc=com` | -| `ldap_bind_user` | no | | LDAP user. | -| `ldap_bind_password` | no | | LDAP password. | +| `ldap_domain` | no | | DNS domain name for DNS SRV lookups (e.g., `example.com`). When omitted, the domain is auto-discovered from the `USERDNSDOMAIN` environment variable (Windows) or the hostname's domain suffix. This setting is used only when `ldap_address` is not provided. | +| `ldap_address` | no | | LDAP server address (eg: `ldap://ds.example.com:389`). If not provided, auto-discovery will be attempted via DNS SRV records and, on Windows, the LOGONSERVER environment variable. | +| `ldap_base_dn` | no | | LDAP base DN (eg: `dc=example,dc=com`). If not provided, auto-discovery will be attempted via rootDSE query or inferred from the server domain. | +| `ldap_bind_user` | no | | LDAP user. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | +| `ldap_bind_password` | no | | LDAP password. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | | `ldap_search_attribute` | yes | `objectGUID` | LDAP attribute to search by. | | `ldap_mapped_attribute` | yes | `cn` | LDAP attribute to map to. | | `ldap_search_time_limit` | no | 30 | LDAP search time limit in seconds. | -| `ldap_ssl`\* | no | 30 | LDAP TLS/SSL connection settings. | +| `ldap_ssl`* | no | | LDAP TLS/SSL connection settings. See [SSL](/reference/metricbeat/configuration-ssl.md). | +| `ad_guid_translation` | no | `auto` | Controls GUID binary conversion for Active Directory attributes. `auto` (default) converts when the LDAP search attribute equals `objectGUID` (case-insensitive). Use `always` to force conversion or `never` to disable it. | | `ignore_missing` | no | false | Ignore errors when the source field is missing. | | `ignore_failure` | no | false | Ignore all errors produced by the processor. | -\* Also see [SSL](/reference/metricbeat/configuration-ssl.md) for a full description of the `ldap_ssl` options. +## Server auto-discovery + +When `ldap_address` is omitted the processor attempts to discover controllers in the following order: + +1. DNS SRV lookups for `_ldaps._tcp` (preferred) and `_ldap._tcp` using the system's native DNS resolver. The processor queries multiple patterns: + - `_ldap._tcp.dc._msdcs.` (Active Directory domain controllers) + - `_ldap._tcp.` (standard domain lookup) + - If no domain is available, bare queries like `_ldap._tcp` which automatically use the system's DNS search suffix configuration (works on Windows, Linux, and macOS) + + The domain used for DNS SRV lookups is determined from: + - The `ldap_domain` configuration option (highest priority) + - Operating-system domain metadata (for example session environment values or Windows domain-join settings) + - The host's fully qualified name when available + - Reverse DNS lookup of the local machine as a last resort + + **Note:** The processor uses Go's standard library `net.LookupSRV()` which leverages the operating system's native DNS resolver. On Windows, this automatically reads DNS servers and search suffixes from the Windows registry, making autodiscovery work correctly even when running as a service without environment variables. SRV records are automatically ordered by priority with weight-based randomization per RFC 2782 to distribute load across available servers. +2. On Windows, the `LOGONSERVER` environment variable. The processor keeps the hostname for TLS validation and may also try the resolved IP as a fallback. + +Each candidate server is tried sequentially until one responds. Likewise, if `ldap_base_dn` is not supplied the client queries the server's rootDSE for `defaultNamingContext`/`namingContexts`, and if that fails, infers the DN from the server's domain name (for example `dc=example,dc=com`). If the searches are slow or you expect a high amount of different key attributes to be found, consider using a cache processor to speed processing: @@ -78,4 +98,3 @@ processors: key_field: winlog.event_data.ObjectGuid value_field: winlog.common_name ``` - diff --git a/docs/reference/packetbeat/processor-translate-guid.md b/docs/reference/packetbeat/processor-translate-guid.md index 6ea74e97af5b..820f7024a62a 100644 --- a/docs/reference/packetbeat/processor-translate-guid.md +++ b/docs/reference/packetbeat/processor-translate-guid.md @@ -6,27 +6,27 @@ applies_to: stack: ga --- -# Translate GUID [processor-translate-guid] +# Translate LDAP Attribute [processor-translate-ldap-attribute] +The `translate_ldap_attribute` processor translates LDAP attributes into friendlier values. The typical use case is converting an Active Directory Global Unique Identifier (GUID) into a human-readable name (for example the object's `cn`). -The `translate_ldap_attribute` processor translates an LDAP attributes between eachother. It is typically used to translate AD Global Unique Identifiers (GUID) into their common names. +Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object's name and these values sometimes appear in logs. -Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object’s name and these values sometimes appear in logs. - -If the search attribute is invalid (malformed) or does not map to any object on the domain then this will result in the processor returning an error unless `ignore_failure` is set. +If the search attribute is invalid (malformed) or does not map to any object on the domain the processor returns an error unless `ignore_failure` is set. The result of this operation is an array of values, given that a single attribute can hold multiple values. -Note: the search attribute is expected to map to a single object. If it doesn’t, no error will be returned, but only results of the first entry will be added to the event. +Note: the search attribute is expected to map to a single object. If multiple entries match, only the first entry's mapped attribute values are returned. ```yaml processors: - translate_ldap_attribute: field: winlog.event_data.ObjectGuid - ldap_address: "ldap://" - ldap_base_dn: "dc=example,dc=com" ignore_missing: true ignore_failure: true + # ldap_domain: "example.com" # Optional - override auto-discovered domain for DNS SRV queries + # ldap_address: "ldap://ds.example.com:389" # Optional - resolve via DNS SRV/LOGONSERVER when omitted + # ldap_base_dn: "dc=example,dc=com" # Optional - discovered via rootDSE or inferred from server domain ``` The `translate_ldap_attribute` processor has the following configuration settings: @@ -35,18 +35,38 @@ The `translate_ldap_attribute` processor has the following configuration setting | --- | --- | --- | --- | | `field` | yes | | Source field containing a GUID. | | `target_field` | no | | Target field for the mapped attribute value. If not set it will be replaced in place. | -| `ldap_address` | yes | | LDAP server address. eg: `ldap://ds.example.com:389` | -| `ldap_base_dn` | yes | | LDAP base DN. eg: `dc=example,dc=com` | -| `ldap_bind_user` | no | | LDAP user. | -| `ldap_bind_password` | no | | LDAP password. | +| `ldap_domain` | no | | DNS domain name for DNS SRV lookups (e.g., `example.com`). When omitted, the domain is auto-discovered from the `USERDNSDOMAIN` environment variable (Windows) or the hostname's domain suffix. This setting is used only when `ldap_address` is not provided. | +| `ldap_address` | no | | LDAP server address (eg: `ldap://ds.example.com:389`). If not provided, auto-discovery will be attempted via DNS SRV records and, on Windows, the LOGONSERVER environment variable. | +| `ldap_base_dn` | no | | LDAP base DN (eg: `dc=example,dc=com`). If not provided, auto-discovery will be attempted via rootDSE query or inferred from the server domain. | +| `ldap_bind_user` | no | | LDAP user. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | +| `ldap_bind_password` | no | | LDAP password. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | | `ldap_search_attribute` | yes | `objectGUID` | LDAP attribute to search by. | | `ldap_mapped_attribute` | yes | `cn` | LDAP attribute to map to. | | `ldap_search_time_limit` | no | 30 | LDAP search time limit in seconds. | -| `ldap_ssl`\* | no | 30 | LDAP TLS/SSL connection settings. | +| `ldap_ssl`* | no | | LDAP TLS/SSL connection settings. See [SSL](/reference/packetbeat/configuration-ssl.md). | +| `ad_guid_translation` | no | `auto` | Controls GUID binary conversion for Active Directory attributes. `auto` (default) converts when the LDAP search attribute equals `objectGUID` (case-insensitive). Use `always` to force conversion or `never` to disable it. | | `ignore_missing` | no | false | Ignore errors when the source field is missing. | | `ignore_failure` | no | false | Ignore all errors produced by the processor. | -\* Also see [SSL](/reference/packetbeat/configuration-ssl.md) for a full description of the `ldap_ssl` options. +## Server auto-discovery + +When `ldap_address` is omitted the processor attempts to discover controllers in the following order: + +1. DNS SRV lookups for `_ldaps._tcp` (preferred) and `_ldap._tcp` using the system's native DNS resolver. The processor queries multiple patterns: + - `_ldap._tcp.dc._msdcs.` (Active Directory domain controllers) + - `_ldap._tcp.` (standard domain lookup) + - If no domain is available, bare queries like `_ldap._tcp` which automatically use the system's DNS search suffix configuration (works on Windows, Linux, and macOS) + + The domain used for DNS SRV lookups is determined from: + - The `ldap_domain` configuration option (highest priority) + - Operating-system domain metadata (for example session environment values or Windows domain-join settings) + - The host's fully qualified name when available + - Reverse DNS lookup of the local machine as a last resort + + **Note:** The processor uses Go's standard library `net.LookupSRV()` which leverages the operating system's native DNS resolver. On Windows, this automatically reads DNS servers and search suffixes from the Windows registry, making autodiscovery work correctly even when running as a service without environment variables. SRV records are automatically ordered by priority with weight-based randomization per RFC 2782 to distribute load across available servers. +2. On Windows, the `LOGONSERVER` environment variable. The processor keeps the hostname for TLS validation and may also try the resolved IP as a fallback. + +Each candidate server is tried sequentially until one responds. Likewise, if `ldap_base_dn` is not supplied the client queries the server's rootDSE for `defaultNamingContext`/`namingContexts`, and if that fails, infers the DN from the server's domain name (for example `dc=example,dc=com`). If the searches are slow or you expect a high amount of different key attributes to be found, consider using a cache processor to speed processing: @@ -78,4 +98,3 @@ processors: key_field: winlog.event_data.ObjectGuid value_field: winlog.common_name ``` - diff --git a/docs/reference/winlogbeat/processor-translate-guid.md b/docs/reference/winlogbeat/processor-translate-guid.md index fe7c1fc483eb..88151c1766aa 100644 --- a/docs/reference/winlogbeat/processor-translate-guid.md +++ b/docs/reference/winlogbeat/processor-translate-guid.md @@ -6,27 +6,27 @@ applies_to: stack: ga --- -# Translate GUID [processor-translate-guid] +# Translate LDAP Attribute [processor-translate-ldap-attribute] +The `translate_ldap_attribute` processor translates LDAP attributes into friendlier values. The typical use case is converting an Active Directory Global Unique Identifier (GUID) into a human-readable name (for example the object's `cn`). -The `translate_ldap_attribute` processor translates an LDAP attributes between eachother. It is typically used to translate AD Global Unique Identifiers (GUID) into their common names. +Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object's name and these values sometimes appear in logs. -Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID’s rather than the object’s name and these values sometimes appear in logs. - -If the search attribute is invalid (malformed) or does not map to any object on the domain then this will result in the processor returning an error unless `ignore_failure` is set. +If the search attribute is invalid (malformed) or does not map to any object on the domain the processor returns an error unless `ignore_failure` is set. The result of this operation is an array of values, given that a single attribute can hold multiple values. -Note: the search attribute is expected to map to a single object. If it doesn’t, no error will be returned, but only results of the first entry will be added to the event. +Note: the search attribute is expected to map to a single object. If multiple entries match, only the first entry's mapped attribute values are returned. ```yaml processors: - translate_ldap_attribute: field: winlog.event_data.ObjectGuid - ldap_address: "ldap://" - ldap_base_dn: "dc=example,dc=com" ignore_missing: true ignore_failure: true + # ldap_domain: "example.com" # Optional - override auto-discovered domain for DNS SRV queries + # ldap_address: "ldap://ds.example.com:389" # Optional - resolve via DNS SRV/LOGONSERVER when omitted + # ldap_base_dn: "dc=example,dc=com" # Optional - discovered via rootDSE or inferred from server domain ``` The `translate_ldap_attribute` processor has the following configuration settings: @@ -35,18 +35,38 @@ The `translate_ldap_attribute` processor has the following configuration setting | --- | --- | --- | --- | | `field` | yes | | Source field containing a GUID. | | `target_field` | no | | Target field for the mapped attribute value. If not set it will be replaced in place. | -| `ldap_address` | yes | | LDAP server address. eg: `ldap://ds.example.com:389` | -| `ldap_base_dn` | yes | | LDAP base DN. eg: `dc=example,dc=com` | -| `ldap_bind_user` | no | | LDAP user. | -| `ldap_bind_password` | no | | LDAP password. | +| `ldap_domain` | no | | DNS domain name for DNS SRV lookups (e.g., `example.com`). When omitted, the domain is auto-discovered from the `USERDNSDOMAIN` environment variable (Windows) or the hostname's domain suffix. This setting is used only when `ldap_address` is not provided. | +| `ldap_address` | no | | LDAP server address (eg: `ldap://ds.example.com:389`). If not provided, auto-discovery will be attempted via DNS SRV records and, on Windows, the LOGONSERVER environment variable. | +| `ldap_base_dn` | no | | LDAP base DN (eg: `dc=example,dc=com`). If not provided, auto-discovery will be attempted via rootDSE query or inferred from the server domain. | +| `ldap_bind_user` | no | | LDAP user. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | +| `ldap_bind_password` | no | | LDAP password. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | | `ldap_search_attribute` | yes | `objectGUID` | LDAP attribute to search by. | | `ldap_mapped_attribute` | yes | `cn` | LDAP attribute to map to. | | `ldap_search_time_limit` | no | 30 | LDAP search time limit in seconds. | -| `ldap_ssl`\* | no | 30 | LDAP TLS/SSL connection settings. | +| `ldap_ssl`* | no | | LDAP TLS/SSL connection settings. See [SSL](/reference/winlogbeat/configuration-ssl.md). | +| `ad_guid_translation` | no | `auto` | Controls GUID binary conversion for Active Directory attributes. `auto` (default) converts when the LDAP search attribute equals `objectGUID` (case-insensitive). Use `always` to force conversion or `never` to disable it. | | `ignore_missing` | no | false | Ignore errors when the source field is missing. | | `ignore_failure` | no | false | Ignore all errors produced by the processor. | -\* Also see [SSL](/reference/winlogbeat/configuration-ssl.md) for a full description of the `ldap_ssl` options. +## Server auto-discovery + +When `ldap_address` is omitted the processor attempts to discover controllers in the following order: + +1. DNS SRV lookups for `_ldaps._tcp` (preferred) and `_ldap._tcp` using the system's native DNS resolver. The processor queries multiple patterns: + - `_ldap._tcp.dc._msdcs.` (Active Directory domain controllers) + - `_ldap._tcp.` (standard domain lookup) + - If no domain is available, bare queries like `_ldap._tcp` which automatically use the system's DNS search suffix configuration (works on Windows, Linux, and macOS) + + The domain used for DNS SRV lookups is determined from: + - The `ldap_domain` configuration option (highest priority) + - Operating-system domain metadata (for example session environment values or Windows domain-join settings) + - The host's fully qualified name when available + - Reverse DNS lookup of the local machine as a last resort + + **Note:** The processor uses Go's standard library `net.LookupSRV()` which leverages the operating system's native DNS resolver. On Windows, this automatically reads DNS servers and search suffixes from the Windows registry, making autodiscovery work correctly even when running as a service without environment variables. SRV records are automatically ordered by priority with weight-based randomization per RFC 2782 to distribute load across available servers. +2. On Windows, the `LOGONSERVER` environment variable. The processor keeps the hostname for TLS validation and may also try the resolved IP as a fallback. + +Each candidate server is tried sequentially until one responds. Likewise, if `ldap_base_dn` is not supplied the client queries the server's rootDSE for `defaultNamingContext`/`namingContexts`, and if that fails, infers the DN from the server's domain name (for example `dc=example,dc=com`). If the searches are slow or you expect a high amount of different key attributes to be found, consider using a cache processor to speed processing: @@ -78,4 +98,3 @@ processors: key_field: winlog.event_data.ObjectGuid value_field: winlog.common_name ``` - diff --git a/filebeat/tests/integration/translate_ldap_attribute_test.go b/filebeat/tests/integration/translate_ldap_attribute_test.go index 7203757a0f98..31c7ed56c39e 100644 --- a/filebeat/tests/integration/translate_ldap_attribute_test.go +++ b/filebeat/tests/integration/translate_ldap_attribute_test.go @@ -21,7 +21,6 @@ package integration import ( "context" - "errors" "fmt" "io" "os" @@ -80,52 +79,295 @@ processors: ` func TestTranslateGUIDWithLDAP(t *testing.T) { - t.Skip("Flaky Test: https://github.com/elastic/beats/issues/42616") startOpenldapContainer(t) - var entryUUID string - require.Eventually(t, func() bool { - var err error - entryUUID, err = getLDAPUserEntryUUID() - return err == nil - }, 10*time.Second, time.Second) + entryUUID := waitForLDAPUser(t, "User1") - filebeat := integration.NewBeat( - t, - "filebeat", - "../../filebeat.test", - ) + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") tempDir := filebeat.TempDir() - // 1. Generate the log file path logFilePath := path.Join(tempDir, "log.log") integration.WriteLogFile(t, logFilePath, 1, false) - // 2. Write configuration file and start Filebeat - filebeat.WriteConfigFile( - fmt.Sprintf(translateguidCfg, logFilePath, tempDir, entryUUID), - ) + filebeat.WriteConfigFile(fmt.Sprintf(translateguidCfg, logFilePath, tempDir, entryUUID)) filebeat.Start() - var outputFile string - require.Eventually(t, func() bool { - outputFiles, err := filepath.Glob(path.Join(tempDir, "output-file-*.ndjson")) - if err != nil { - return false - } - if len(outputFiles) != 1 { - return false + outputFile := waitForOutputFile(t, tempDir) + + filebeat.WaitFileContains(outputFile, fmt.Sprintf(`"guid":"%s"`, entryUUID), 20*time.Second) + filebeat.WaitFileContains(outputFile, `"common_name":["User1","user01"]`, 5*time.Second) +} + +const translateMultipleCfg = ` +filebeat.inputs: + - type: filestream + id: "test-translateMultipleCfg" + file_identity.native: ~ + prospector.scanner.fingerprint.enabled: false + paths: + - %s + +queue.mem: + flush.min_events: 1 + flush.timeout: 0.1s + +path.home: %s + +output.file: + path: ${path.home} + filename: "output-file" + +logging: + metrics: + enabled: false + +processors: + - decode_json_fields: + fields: ["message"] + target: "" + - translate_ldap_attribute: + field: guid + target_field: common_name + ldap_address: 'ldap://localhost:1389' + ldap_base_dn: 'dc=example,dc=org' + ldap_bind_user: 'cn=admin,dc=example,dc=org' + ldap_bind_password: 'adminpassword' + ldap_search_attribute: 'entryUUID' + ignore_missing: true + ignore_failure: true +` + +func TestTranslateGUIDWithMultipleCallsAndFailures(t *testing.T) { + startOpenldapContainer(t) + + entryUUID1 := waitForLDAPUser(t, "User1") + entryUUID2 := waitForLDAPUser(t, "User2") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + tempDir := filebeat.TempDir() + logFilePath := path.Join(tempDir, "log.log") + + entries := []string{ + fmt.Sprintf(`{"guid":"%s","message":"valid entry 1"}`, entryUUID1), + `{"guid":"00000000-0000-0000-0000-000000000000","message":"invalid entry 1"}`, + fmt.Sprintf(`{"guid":"%s","message":"valid entry 2"}`, entryUUID2), + `{"guid":"11111111-1111-1111-1111-111111111111","message":"invalid entry 2"}`, + fmt.Sprintf(`{"guid":"%s","message":"valid entry 3"}`, entryUUID1), + `{"guid":"22222222-2222-2222-2222-222222222222","message":"invalid entry 3"}`, + fmt.Sprintf(`{"guid":"%s","message":"valid entry 4"}`, entryUUID2), + `{"message":"no guid field"}`, + fmt.Sprintf(`{"guid":"%s","message":"valid entry 5"}`, entryUUID1), + `{"guid":"33333333-3333-3333-3333-333333333333","message":"invalid entry 4"}`, + } + + logFile, err := os.Create(logFilePath) + require.NoError(t, err) + for _, entry := range entries { + _, err := logFile.WriteString(entry + "\n") + require.NoError(t, err) + } + logFile.Close() + + filebeat.WriteConfigFile(fmt.Sprintf(translateMultipleCfg, logFilePath, tempDir)) + filebeat.Start() + + outputFile := waitForOutputFile(t, tempDir) + + filebeat.WaitFileContains(outputFile, fmt.Sprintf(`"guid":"%s"`, entryUUID1), 30*time.Second) + filebeat.WaitFileContains(outputFile, `"common_name":["User1","user01"]`, 5*time.Second) + filebeat.WaitFileContains(outputFile, fmt.Sprintf(`"guid":"%s"`, entryUUID2), 5*time.Second) + filebeat.WaitFileContains(outputFile, `"common_name":["User2","user02"]`, 5*time.Second) + + filebeat.WaitFileContains(outputFile, `"guid":"00000000-0000-0000-0000-000000000000"`, 5*time.Second) + filebeat.WaitFileContains(outputFile, `"guid":"33333333-3333-3333-3333-333333333333"`, 5*time.Second) + filebeat.WaitFileContains(outputFile, `no guid field`, 5*time.Second) +} + +const translateConcurrentCfg = ` +filebeat.inputs: + - type: filestream + id: "test-translateConcurrentCfg-1" + file_identity.native: ~ + prospector.scanner.fingerprint.enabled: false + paths: + - %s + + - type: filestream + id: "test-translateConcurrentCfg-2" + file_identity.native: ~ + prospector.scanner.fingerprint.enabled: false + paths: + - %s + +queue.mem: + flush.min_events: 1 + flush.timeout: 0.1s + +path.home: %s + +output.file: + path: ${path.home} + filename: "output-file" + +logging: + metrics: + enabled: false + +processors: + - decode_json_fields: + fields: ["message"] + target: "" + - translate_ldap_attribute: + field: guid + target_field: common_name + ldap_address: 'ldap://localhost:1389' + ldap_base_dn: 'dc=example,dc=org' + ldap_bind_user: 'cn=admin,dc=example,dc=org' + ldap_bind_password: 'adminpassword' + ldap_search_attribute: 'entryUUID' + ignore_missing: true + ignore_failure: true +` + +func TestTranslateGUIDWithConcurrentCalls(t *testing.T) { + startOpenldapContainer(t) + + entryUUID1 := waitForLDAPUser(t, "User1") + entryUUID2 := waitForLDAPUser(t, "User2") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + tempDir := filebeat.TempDir() + + logFilePath1 := path.Join(tempDir, "log1.log") + logFilePath2 := path.Join(tempDir, "log2.log") + + entries1 := []string{ + fmt.Sprintf(`{"guid":"%s","message":"concurrent file 1 - entry 1"}`, entryUUID1), + fmt.Sprintf(`{"guid":"%s","message":"concurrent file 1 - entry 2"}`, entryUUID2), + fmt.Sprintf(`{"guid":"%s","message":"concurrent file 1 - entry 3"}`, entryUUID1), + fmt.Sprintf(`{"guid":"%s","message":"concurrent file 1 - entry 4"}`, entryUUID2), + fmt.Sprintf(`{"guid":"%s","message":"concurrent file 1 - entry 5"}`, entryUUID1), + } + + entries2 := []string{ + fmt.Sprintf(`{"guid":"%s","message":"concurrent file 2 - entry 1"}`, entryUUID2), + fmt.Sprintf(`{"guid":"%s","message":"concurrent file 2 - entry 2"}`, entryUUID1), + fmt.Sprintf(`{"guid":"%s","message":"concurrent file 2 - entry 3"}`, entryUUID2), + fmt.Sprintf(`{"guid":"%s","message":"concurrent file 2 - entry 4"}`, entryUUID1), + fmt.Sprintf(`{"guid":"%s","message":"concurrent file 2 - entry 5"}`, entryUUID2), + } + + logFile1, err := os.Create(logFilePath1) + require.NoError(t, err) + for _, entry := range entries1 { + _, err := logFile1.WriteString(entry + "\n") + require.NoError(t, err) + } + logFile1.Close() + + logFile2, err := os.Create(logFilePath2) + require.NoError(t, err) + for _, entry := range entries2 { + _, err := logFile2.WriteString(entry + "\n") + require.NoError(t, err) + } + logFile2.Close() + + filebeat.WriteConfigFile(fmt.Sprintf(translateConcurrentCfg, logFilePath1, logFilePath2, tempDir)) + filebeat.Start() + + outputFile := waitForOutputFile(t, tempDir) + + filebeat.WaitFileContains(outputFile, `concurrent file 1 - entry 1`, 30*time.Second) + filebeat.WaitFileContains(outputFile, `concurrent file 2 - entry 1`, 5*time.Second) + filebeat.WaitFileContains(outputFile, `"common_name":["User1","user01"]`, 5*time.Second) + filebeat.WaitFileContains(outputFile, `"common_name":["User2","user02"]`, 5*time.Second) + filebeat.WaitFileContains(outputFile, `concurrent file 1 - entry 5`, 5*time.Second) + filebeat.WaitFileContains(outputFile, `concurrent file 2 - entry 5`, 5*time.Second) +} + +const translateConcurrentWorkersCfg = ` +filebeat.inputs: + - type: filestream + id: "test-translateConcurrentWorkersCfg" + file_identity.native: ~ + prospector.scanner.fingerprint.enabled: false + paths: + - %s + +queue.mem: + flush.min_events: 1 + flush.timeout: 0.1s + +# Enable multiple pipeline workers for concurrent processing +queue.mem.events: 4096 +pipeline.workers: 4 + +path.home: %s + +output.file: + path: ${path.home} + filename: "output-file" + +logging: + metrics: + enabled: false + +processors: + - decode_json_fields: + fields: ["message"] + target: "" + - translate_ldap_attribute: + field: guid + target_field: common_name + ldap_address: 'ldap://localhost:1389' + ldap_base_dn: 'dc=example,dc=org' + ldap_bind_user: 'cn=admin,dc=example,dc=org' + ldap_bind_password: 'adminpassword' + ldap_search_attribute: 'entryUUID' + ignore_missing: true + ignore_failure: true +` + +func TestTranslateGUIDWithConcurrentWorkersInSameInput(t *testing.T) { + startOpenldapContainer(t) + + entryUUID1 := waitForLDAPUser(t, "User1") + entryUUID2 := waitForLDAPUser(t, "User2") + + filebeat := integration.NewBeat(t, "filebeat", "../../filebeat.test") + tempDir := filebeat.TempDir() + logFilePath := path.Join(tempDir, "log.log") + + entries := []string{} + for i := 1; i <= 20; i++ { + if i%2 == 0 { + entries = append(entries, fmt.Sprintf(`{"guid":"%s","message":"worker test entry %d"}`, entryUUID1, i)) + } else { + entries = append(entries, fmt.Sprintf(`{"guid":"%s","message":"worker test entry %d"}`, entryUUID2, i)) } - outputFile = outputFiles[0] - return true - }, 10*time.Second, time.Second) + } - // 3. Wait for the event with the expected translated guid - filebeat.WaitFileContains( - outputFile, - fmt.Sprintf(`"fields":{"guid":"%s","common_name":["User1","user01"]}`, entryUUID), - 20*time.Second, - ) + logFile, err := os.Create(logFilePath) + require.NoError(t, err) + for _, entry := range entries { + _, err := logFile.WriteString(entry + "\n") + require.NoError(t, err) + } + logFile.Close() + + filebeat.WriteConfigFile(fmt.Sprintf(translateConcurrentWorkersCfg, logFilePath, tempDir)) + filebeat.Start() + + outputFile := waitForOutputFile(t, tempDir) + + filebeat.WaitFileContains(outputFile, `worker test entry 1`, 30*time.Second) + filebeat.WaitFileContains(outputFile, `worker test entry 20`, 10*time.Second) + filebeat.WaitFileContains(outputFile, `"common_name":["User1","user01"]`, 5*time.Second) + filebeat.WaitFileContains(outputFile, `"common_name":["User2","user02"]`, 5*time.Second) + filebeat.WaitFileContains(outputFile, `worker test entry 5`, 5*time.Second) + filebeat.WaitFileContains(outputFile, `worker test entry 10`, 5*time.Second) + filebeat.WaitFileContains(outputFile, `worker test entry 15`, 5*time.Second) } func startOpenldapContainer(t *testing.T) { @@ -135,7 +377,7 @@ func startOpenldapContainer(t *testing.T) { t.Fatal(err) } - reader, err := c.ImagePull(ctx, "bitnami/openldap:2", image.PullOptions{}) + reader, err := c.ImagePull(ctx, "osixia/openldap:1.5.0", image.PullOptions{}) if err != nil { t.Fatal(err) } @@ -146,20 +388,19 @@ func startOpenldapContainer(t *testing.T) { resp, err := c.ContainerCreate(ctx, &container.Config{ - Image: "bitnami/openldap:2", + Image: "osixia/openldap:1.5.0", ExposedPorts: nat.PortSet{ - "1389/tcp": struct{}{}, + "389/tcp": struct{}{}, }, Env: []string{ - "LDAP_URI=ldap://openldap:1389", - "LDAP_BASE=dc=example,dc=org", - "LDAP_BIND_DN=cn=admin,dc=example,dc=org", - "LDAP_BIND_PASSWORD=adminpassword", + "LDAP_ORGANISATION=example", + "LDAP_DOMAIN=example.org", + "LDAP_ADMIN_PASSWORD=adminpassword", }, }, &container.HostConfig{ PortBindings: nat.PortMap{ - "1389/tcp": []nat.PortBinding{ + "389/tcp": []nat.PortBinding{ { HostIP: "0.0.0.0", HostPort: "1389", @@ -181,25 +422,66 @@ func startOpenldapContainer(t *testing.T) { t.Error(err) } }) + + require.Eventually(t, func() bool { + return addTestUserToLDAP() == nil + }, 30*time.Second, time.Second, "Failed to add test users to LDAP") } -func getLDAPUserEntryUUID() (string, error) { - // Connect to the LDAP server +func connectToLDAP() (*ldap.Conn, error) { l, err := ldap.DialURL("ldap://localhost:1389") if err != nil { - return "", fmt.Errorf("failed to connect to LDAP server: %w", err) + return nil, fmt.Errorf("failed to connect to LDAP server: %w", err) + } + + if err = l.Bind("cn=admin,dc=example,dc=org", "adminpassword"); err != nil { + l.Close() + return nil, fmt.Errorf("failed to bind to LDAP server: %w", err) + } + + return l, nil +} + +func addTestUserToLDAP() error { + l, err := connectToLDAP() + if err != nil { + return err } defer l.Close() - err = l.Bind("cn=admin,dc=example,dc=org", "adminpassword") + users := []struct { + cn, uid string + }{ + {"User1", "user01"}, + {"User2", "user02"}, + } + + for _, user := range users { + addRequest := ldap.NewAddRequest(fmt.Sprintf("cn=%s,dc=example,dc=org", user.cn), nil) + addRequest.Attribute("objectClass", []string{"inetOrgPerson", "organizationalPerson", "person", "top"}) + addRequest.Attribute("cn", []string{user.cn, user.uid}) + addRequest.Attribute("sn", []string{user.cn}) + addRequest.Attribute("uid", []string{user.uid}) + + if err := l.Add(addRequest); err != nil { + return fmt.Errorf("failed to add test user %s: %w", user.cn, err) + } + } + + return nil +} + +func getLDAPUserEntryUUID(username string) (string, error) { + l, err := connectToLDAP() if err != nil { - return "", fmt.Errorf("failed to bind to LDAP server: %w", err) + return "", err } + defer l.Close() searchRequest := ldap.NewSearchRequest( "dc=example,dc=org", ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 1, 0, false, - "(cn=User1)", []string{"entryUUID"}, nil, + fmt.Sprintf("(cn=%s)", username), []string{"entryUUID"}, nil, ) sr, err := l.Search(searchRequest) @@ -207,14 +489,35 @@ func getLDAPUserEntryUUID() (string, error) { return "", fmt.Errorf("failed to execute search: %w", err) } - // Process search results if len(sr.Entries) == 0 { - return "", errors.New("no entries found for the specified username.") + return "", fmt.Errorf("no entries found for username: %s", username) } - entry := sr.Entries[0] - entryUUID := entry.GetAttributeValue("entryUUID") + entryUUID := sr.Entries[0].GetAttributeValue("entryUUID") if entryUUID == "" { - return "", errors.New("entryUUID is empty") + return "", fmt.Errorf("entryUUID is empty for username: %s", username) } return entryUUID, nil } + +func waitForLDAPUser(t *testing.T, username string) string { + var entryUUID string + require.Eventually(t, func() bool { + var err error + entryUUID, err = getLDAPUserEntryUUID(username) + return err == nil + }, 10*time.Second, time.Second) + return entryUUID +} + +func waitForOutputFile(t *testing.T, tempDir string) string { + var outputFile string + require.Eventually(t, func() bool { + outputFiles, err := filepath.Glob(path.Join(tempDir, "output-file-*.ndjson")) + if err != nil || len(outputFiles) != 1 { + return false + } + outputFile = outputFiles[0] + return true + }, 10*time.Second, time.Second) + return outputFile +} diff --git a/go.mod b/go.mod index 31a5b1b4f286..7079b5e49398 100644 --- a/go.mod +++ b/go.mod @@ -287,6 +287,7 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect github.com/VictoriaMetrics/easyproto v0.1.4 // indirect + github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apache/arrow/go/v15 v15.0.2 // indirect diff --git a/libbeat/processors/translate_ldap_attribute/config.go b/libbeat/processors/translate_ldap_attribute/config.go index 4a669f582de2..ce59e5b4da02 100644 --- a/libbeat/processors/translate_ldap_attribute/config.go +++ b/libbeat/processors/translate_ldap_attribute/config.go @@ -20,14 +20,24 @@ package translate_ldap_attribute import ( + "fmt" + "strings" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" ) +const ( + guidTranslationAuto = "auto" + guidTranslationAlways = "always" + guidTranslationNever = "never" +) + type config struct { Field string `config:"field" validate:"required"` TargetField string `config:"target_field"` - LDAPAddress string `config:"ldap_address" validate:"required"` - LDAPBaseDN string `config:"ldap_base_dn" validate:"required"` + LDAPDomain string `config:"ldap_domain"` + LDAPAddress string `config:"ldap_address"` + LDAPBaseDN string `config:"ldap_base_dn"` LDAPBindUser string `config:"ldap_bind_user"` LDAPBindPassword string `config:"ldap_bind_password"` LDAPSearchAttribute string `config:"ldap_search_attribute" validate:"required"` @@ -35,6 +45,16 @@ type config struct { LDAPSearchTimeLimit int `config:"ldap_search_time_limit"` LDAPTLS *tlscommon.Config `config:"ldap_ssl"` + // ADGUIDTranslation controls when GUID values get converted to the binary form + // expected by Active Directory. We no longer rely on server detection; the + // auto mode simply checks whether the configured search attribute is named + // objectGUID (case-insensitive). + // Supported values: + // "auto" (default): Convert when LDAP search attribute equals objectGUID + // "always": Always apply GUID conversion regardless of attribute name + // "never" : Never apply GUID conversion + ADGUIDTranslation string `config:"ad_guid_translation"` + IgnoreMissing bool `config:"ignore_missing"` IgnoreFailure bool `config:"ignore_failure"` } @@ -43,5 +63,21 @@ func defaultConfig() config { return config{ LDAPSearchAttribute: "objectGUID", LDAPMappedAttribute: "cn", - LDAPSearchTimeLimit: 30} + LDAPSearchTimeLimit: 30, + ADGUIDTranslation: guidTranslationAuto, + } +} + +func (c *config) validate() error { + switch strings.ToLower(strings.TrimSpace(c.ADGUIDTranslation)) { + case "", guidTranslationAuto: + c.ADGUIDTranslation = guidTranslationAuto + case guidTranslationAlways: + c.ADGUIDTranslation = guidTranslationAlways + case guidTranslationNever: + c.ADGUIDTranslation = guidTranslationNever + default: + return fmt.Errorf("invalid ad_guid_translation value %q (expected auto|always|never)", c.ADGUIDTranslation) + } + return nil } diff --git a/libbeat/processors/translate_ldap_attribute/config_test.go b/libbeat/processors/translate_ldap_attribute/config_test.go new file mode 100644 index 000000000000..40be593ed042 --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/config_test.go @@ -0,0 +1,57 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !requirefips + +package translate_ldap_attribute + +import "testing" + +func TestConfigValidate(t *testing.T) { + tests := []struct { + name string + value string + expect string + expectError bool + }{ + {name: "empty defaults to auto", value: "", expect: guidTranslationAuto}, + {name: "explicit auto", value: "auto", expect: guidTranslationAuto}, + {name: "explicit always", value: "always", expect: guidTranslationAlways}, + {name: "case insensitive", value: " NEVER ", expect: guidTranslationNever}, + {name: "invalid", value: "sometimes", expectError: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := defaultConfig() + cfg.ADGUIDTranslation = tt.value + err := cfg.validate() + if tt.expectError { + if err == nil { + t.Fatalf("expected error") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.ADGUIDTranslation != tt.expect { + t.Fatalf("expected %q, got %q", tt.expect, cfg.ADGUIDTranslation) + } + }) + } +} diff --git a/libbeat/processors/translate_ldap_attribute/discovery.go b/libbeat/processors/translate_ldap_attribute/discovery.go new file mode 100644 index 000000000000..28bba06e6fdc --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/discovery.go @@ -0,0 +1,276 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !requirefips + +package translate_ldap_attribute + +import ( + "errors" + "fmt" + "net" + "os" + "runtime" + "strings" + + "github.com/elastic/elastic-agent-libs/logp" +) + +var ( + // errNoLDAPServerFound is returned when no LDAP server can be discovered + errNoLDAPServerFound = errors.New("no LDAP server found via DNS SRV or system configuration") + + // resolveTCPAddr allows tests to stub DNS resolution + resolveTCPAddr = net.ResolveTCPAddr +) + +// discoverLDAPAddress attempts to auto-discover the LDAP server address. +// It returns a list of candidate addresses sorted by preference (LDAPS over LDAP, SRV over LOGONSERVER). +// The caller should attempt to connect to each address in order until one succeeds. +func discoverLDAPAddress(configDomain string, log *logp.Logger) ([]string, error) { + log.Debug("attempting LDAP server auto-discovery") + + domain := normalizeDomain(configDomain) + if domain == "" { + domain = discoverDomainName(log) + } + + var candidates []string + + // 1. Primary: DNS SRV Lookup (LDAPS, then LDAP) + candidates = append(candidates, lookupSRVServers(domain, true, log)...) + candidates = append(candidates, lookupSRVServers(domain, false, log)...) + + if len(candidates) > 0 { + return candidates, nil + } + + // 2. Windows Fallback: LOGONSERVER environment variable + log.Debug("attempting discovery via LOGONSERVER environment variable") + candidates = append(candidates, findLogonServer(true, log)...) + candidates = append(candidates, findLogonServer(false, log)...) + + if len(candidates) == 0 { + log.Warnw("no LDAP servers discovered", "dns_srv_attempted", true, "logonserver_attempted", runtime.GOOS == "windows") + return nil, errNoLDAPServerFound + } + + log.Infow("LDAP server auto-discovery completed", "total_candidates", len(candidates), "candidates", candidates) + return candidates, nil +} + +type domainSource struct { + name string + getter func() string +} + +// discoverDomainName attempts to discover the DNS domain name from various sources in priority order. +func discoverDomainName(log *logp.Logger) string { + + sources := []domainSource{ + { + name: "USERDNSDOMAIN", + getter: func() string { return os.Getenv("USERDNSDOMAIN") }, + }, + } + + if h, err := os.Hostname(); err == nil && h != "" { + sources = append(sources, + domainSource{ + name: "hostname", + getter: func() string { + if !strings.Contains(h, ".") { + return "" + } + parts := strings.SplitN(h, ".", 2) + if len(parts) == 2 { + return parts[1] + } + return "" + }, + }, + domainSource{ + name: "reverse_dns", + getter: func() string { return discoverDomainViaReverseDNS(h, log) }, + }, + ) + } else { + log.Debugw("failed to read hostname", "error", err) + } + + if runtime.GOOS == "windows" { + sources = append(sources, domainSource{ + name: "windows_registry", + getter: func() string { return discoverDomainFromRegistry(log) }, + }) + } + + triedSources := make([]string, 0, len(sources)) + for _, source := range sources { + triedSources = append(triedSources, source.name) + domain := normalizeDomain(source.getter()) + if domain != "" { + log.Infow("discovered domain name", "source", source.name, "domain", domain) + return domain + } + } + + log.Debugw("no domain name discovered", "sources_tried", triedSources) + return "" +} + +// normalizeDomain trims and lower-cases a domain value. +func normalizeDomain(domain string) string { + return strings.ToLower(strings.TrimSpace(domain)) +} + +// discoverDomainViaReverseDNS performs reverse DNS lookup to discover the domain. +// It resolves the hostname to IP addresses, then does reverse lookup to get FQDN. +func discoverDomainViaReverseDNS(hostname string, log *logp.Logger) string { + addrs, err := net.LookupHost(hostname) + if err != nil { + log.Debugw("failed to resolve hostname for reverse DNS lookup", "hostname", hostname, "error", err) + return "" + } + + for _, addr := range addrs { + if strings.Contains(addr, "%") { + continue + } + + names, err := net.LookupAddr(addr) + if err != nil { + log.Debugw("reverse DNS lookup failed", "ip", addr, "error", err) + continue + } + + for _, name := range names { + name = strings.TrimSuffix(name, ".") + if !strings.Contains(name, ".") { + continue + } + parts := strings.SplitN(name, ".", 2) + if len(parts) != 2 { + continue + } + log.Infow("discovered domain name via reverse DNS", "fqdn", name, "domain", parts[1], "ip", addr) + return parts[1] + } + } + + log.Debugw("reverse DNS lookup did not yield domain", "hostname", hostname, "addresses_tried", len(addrs)) + return "" +} + +// lookupSRVServers performs DNS SRV lookups for the provided domain using the Go resolver. +func lookupSRVServers(domain string, useTLS bool, log *logp.Logger) []string { + service := "ldap" + scheme := "ldap" + if useTLS { + service = "ldaps" + scheme = "ldaps" + } + + queries := buildSRVQueries(service, domain, log) + var netSRVs []*net.SRV + var successQuery string + + for _, query := range queries { + log.Infow("executing DNS SRV lookup", "query", query, "service", service) + _, records, err := net.LookupSRV("", "", query) + if err == nil && len(records) > 0 { + log.Infow("DNS SRV lookup succeeded", "query", query, "record_count", len(records)) + netSRVs = records + successQuery = query + break + } + log.Debugw("DNS SRV lookup failed", "query", query, "error", err) + } + + if len(netSRVs) == 0 { + log.Warnw("all DNS SRV lookup attempts failed", "domain", domain, "queries_tried", len(queries)) + return nil + } + + var addresses []string + for _, addr := range netSRVs { + target := strings.TrimSuffix(addr.Target, ".") + addresses = append(addresses, fmt.Sprintf("%s://%s:%d", scheme, target, addr.Port)) + } + + log.Infow("discovered servers via DNS SRV", "scheme", scheme, "query", successQuery, "count", len(addresses), "addresses", addresses) + return addresses +} + +func buildSRVQueries(service, domain string, log *logp.Logger) []string { + if domain != "" { + domain = fmt.Sprintf(".%s", domain) + } + return []string{ + fmt.Sprintf("_%s._tcp.dc._msdcs%s", service, domain), + fmt.Sprintf("_%s._tcp%s", service, domain), + } +} + +// findLogonServer is a simplified wrapper for LOGONSERVER lookup. +// It attempts to resolve the NetBIOS name to an IP address before returning the address. +func findLogonServer(useTLS bool, log *logp.Logger) []string { + logonServer := os.Getenv("LOGONSERVER") + if logonServer == "" { + log.Debug("LOGONSERVER environment variable not set") + return nil + } + + // Remove leading backslashes (Windows format: \\SERVERNAME) + serverName := strings.TrimPrefix(logonServer, "\\\\") + serverName = strings.TrimPrefix(serverName, "\\") + + if serverName == "" { + log.Debugw("invalid LOGONSERVER format", "value", logonServer) + return nil + } + + scheme := "ldap" + port := 389 + if useTLS { + scheme = "ldaps" + port = 636 + } + + // Attempt to resolve the NetBIOS name to a resolvable IP/Hostname. + addressToResolve := net.JoinHostPort(serverName, fmt.Sprintf("%d", port)) + + var resolvedIP string + resolvedAddr, err := resolveTCPAddr("tcp", addressToResolve) + if err != nil { + log.Debugw("failed to resolve LOGONSERVER address", "server_name", serverName, "port", port, "error", err) + } else if resolvedAddr.IP != nil { + resolvedIP = resolvedAddr.IP.String() + } + + var addresses []string + hostURL := fmt.Sprintf("%s://%s:%d", scheme, serverName, port) + addresses = append(addresses, hostURL) + + if resolvedIP != "" && !strings.EqualFold(resolvedIP, serverName) { + addresses = append(addresses, fmt.Sprintf("%s://%s:%d", scheme, resolvedIP, port)) + } + + log.Infow("discovered server via LOGONSERVER", "addresses", addresses, "original_name", serverName) + + return addresses +} diff --git a/libbeat/processors/translate_ldap_attribute/discovery_other.go b/libbeat/processors/translate_ldap_attribute/discovery_other.go new file mode 100644 index 000000000000..25c62663b5df --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/discovery_other.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !windows || requirefips + +package translate_ldap_attribute + +import ( + "github.com/elastic/elastic-agent-libs/logp" +) + +// discoverDomainFromRegistry is a no-op on non-Windows platforms. +func discoverDomainFromRegistry(log *logp.Logger) string { + return "" +} diff --git a/libbeat/processors/translate_ldap_attribute/discovery_test.go b/libbeat/processors/translate_ldap_attribute/discovery_test.go new file mode 100644 index 000000000000..6d2e3e3a781f --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/discovery_test.go @@ -0,0 +1,130 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !requirefips + +package translate_ldap_attribute + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/logp" +) + +func TestFindLogonServerPreservesHostname(t *testing.T) { + t.Setenv("LOGONSERVER", "\\\\DC01") + + originalResolver := resolveTCPAddr + resolveTCPAddr = func(network, address string) (*net.TCPAddr, error) { + return &net.TCPAddr{IP: net.ParseIP("192.0.2.10")}, nil + } + t.Cleanup(func() { resolveTCPAddr = originalResolver }) + + log := logp.NewLogger("test") + addresses := findLogonServer(true, log) + require.Len(t, addresses, 2) + assert.Equal(t, "ldaps://DC01:636", addresses[0]) + assert.Equal(t, "ldaps://192.0.2.10:636", addresses[1]) +} + +func TestFindLogonServerFallsBackWithoutResolution(t *testing.T) { + t.Setenv("LOGONSERVER", "\\\\DC02") + + originalResolver := resolveTCPAddr + resolveTCPAddr = func(network, address string) (*net.TCPAddr, error) { + return nil, assert.AnError + } + t.Cleanup(func() { resolveTCPAddr = originalResolver }) + + log := logp.NewLogger("test") + addresses := findLogonServer(false, log) + require.Len(t, addresses, 1) + assert.Equal(t, "ldap://DC02:389", addresses[0]) +} + +func TestInferBaseDNFromAddress(t *testing.T) { + tests := []struct { + name string + address string + expectDN string + expectErr bool + }{ + { + name: "Skip first label with 3 parts", + address: "ldap://dc1.example.com:389", + expectDN: "dc=example,dc=com", + expectErr: false, + }, + { + name: "Keep multi-level domain when skipping host", + address: "ldaps://corp.eu.example.com", + expectDN: "dc=eu,dc=example,dc=com", + expectErr: false, + }, + { + name: "Two part domain no skip", + address: "ldaps://example.com:636", + expectDN: "dc=example,dc=com", + expectErr: false, + }, + { + name: "Multi part domain co.uk", + address: "ldap://auth.example.co.uk:389", + expectDN: "dc=example,dc=co,dc=uk", + expectErr: false, + }, + { + name: "Hostname only (no domain)", + address: "ldap://localhost:389", + expectErr: true, + }, + { + name: "IPv4 address (cannot infer)", + address: "ldaps://192.168.1.10:636", + expectErr: true, + }, + { + name: "IPv6 address (cannot infer)", + address: "ldap://[2001:db8::1]:389", + expectErr: true, + }, + { + name: "Normalizes case and trailing dots", + address: "LDAPS://CORP.EXAMPLE.COM.:636", + expectDN: "dc=example,dc=com", + expectErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := &ldapClient{ldapConfig: &ldapConfig{address: tt.address}, log: logp.NewLogger("test")} + err := client.inferBaseDNFromAddress() + if tt.expectErr { + require.Error(t, err) + assert.Empty(t, client.baseDN) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expectDN, client.baseDN) + } + }) + } +} diff --git a/libbeat/processors/translate_ldap_attribute/discovery_windows.go b/libbeat/processors/translate_ldap_attribute/discovery_windows.go new file mode 100644 index 000000000000..b9f144fa3cdc --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/discovery_windows.go @@ -0,0 +1,54 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build windows && !requirefips + +package translate_ldap_attribute + +import ( + "github.com/elastic/elastic-agent-libs/logp" + "golang.org/x/sys/windows/registry" +) + +// discoverDomainFromRegistry attempts to read the Windows domain from the registry. +// This works in service contexts where environment variables may not be available. +func discoverDomainFromRegistry(log *logp.Logger) string { + // Try primary location: HKLM\SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Domain + k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters`, registry.QUERY_VALUE) + if err != nil { + log.Debugw("failed to open registry key for domain lookup", "error", err) + return "" + } + defer k.Close() + + // Try "Domain" value first + domain, _, err := k.GetStringValue("Domain") + if err == nil && domain != "" { + log.Infow("discovered domain name from Windows registry", "key", "Domain", "domain", domain) + return domain + } + + // Fallback to "DhcpDomain" if Domain is not set + domain, _, err = k.GetStringValue("DhcpDomain") + if err == nil && domain != "" { + log.Infow("discovered domain name from Windows registry", "key", "DhcpDomain", "domain", domain) + return domain + } + + log.Debugw("no domain found in Windows registry", "keys_checked", []string{"Domain", "DhcpDomain"}) + return "" +} diff --git a/libbeat/processors/translate_ldap_attribute/docs/translate_ldap_attribute.asciidoc b/libbeat/processors/translate_ldap_attribute/docs/translate_ldap_attribute.asciidoc index aff1125f43a2..ad3907f4eecc 100644 --- a/libbeat/processors/translate_ldap_attribute/docs/translate_ldap_attribute.asciidoc +++ b/libbeat/processors/translate_ldap_attribute/docs/translate_ldap_attribute.asciidoc @@ -1,38 +1,37 @@ -[[processor-translate-guid]] -=== Translate GUID +[[processor-translate-ldap-attribute]] +=== Translate LDAP Attribute ++++ translate_ldap_attribute ++++ -The `translate_ldap_attribute` processor translates an LDAP attributes between eachother. -It is typically used to translate AD Global Unique Identifiers (GUID) -into their common names. +The `translate_ldap_attribute` processor translates LDAP attributes into friendlier +values. The typical use case is converting an Active Directory Global Unique +Identifier (GUID) into a human-readable name (for example the object's `cn`). Every object on an Active Directory or an LDAP server is issued a GUID. Internal processes refer to their GUID's rather than the object's name and these values sometimes appear in logs. -If the search attribute is invalid (malformed) or does not map to any object on the domain -then this will result in the processor returning an error unless `ignore_failure` -is set. +If the search attribute is invalid (malformed) or does not map to any object on the +domain the processor returns an error unless `ignore_failure` is set. The result of this operation is an array of values, given that a single attribute can hold multiple values. -Note: the search attribute is expected to map to a single object. If it doesn't, -no error will be returned, but only results of the first entry will be added -to the event. +Note: the search attribute is expected to map to a single object. If multiple +entries match, only the first entry's mapped attribute values are returned. [source,yaml] ---- processors: - translate_ldap_attribute: field: winlog.event_data.ObjectGuid - ldap_address: "ldap://" - ldap_base_dn: "dc=example,dc=com" ignore_missing: true ignore_failure: true + # ldap_domain: "example.com" # Optional - override auto-discovered domain for DNS SRV queries + # ldap_address: "ldap://ds.example.com:389" # Optional - resolve via DNS SRV/LOGONSERVER when omitted + # ldap_base_dn: "dc=example,dc=com" # Optional - discovered via rootDSE or inferred from server domain ---- The `translate_ldap_attribute` processor has the following configuration settings: @@ -43,20 +42,54 @@ The `translate_ldap_attribute` processor has the following configuration setting | Name | Required | Default | Description | `field` | yes | | Source field containing a GUID. | `target_field` | no | | Target field for the mapped attribute value. If not set it will be replaced in place. -| `ldap_address` | yes | | LDAP server address. eg: `ldap://ds.example.com:389` -| `ldap_base_dn` | yes | | LDAP base DN. eg: `dc=example,dc=com` -| `ldap_bind_user` | no | | LDAP user. -| `ldap_bind_password` | no | | LDAP password. +| `ldap_domain` | no | | DNS domain name for DNS SRV lookups (eg: `example.com`). When omitted, the domain is auto-discovered from the `USERDNSDOMAIN` environment variable (Windows) or hostname. This setting is only used when `ldap_address` is not provided. +| `ldap_address` | no | | LDAP server address (eg: `ldap://ds.example.com:389`). If not provided, auto-discovery will be attempted via DNS SRV records and, on Windows, the LOGONSERVER environment variable. +| `ldap_base_dn` | no | | LDAP base DN (eg: `dc=example,dc=com`). If not provided, auto-discovery will be attempted via rootDSE query or inferred from the server domain. +| `ldap_bind_user` | no | | LDAP user. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. +| `ldap_bind_password` | no | | LDAP password. If both `ldap_bind_user` and `ldap_bind_password` are omitted, the processor will attempt Windows SSPI authentication (on Windows) using the current process user's credentials, or fall back to an unauthenticated bind. | `ldap_search_attribute` | yes | `objectGUID` | LDAP attribute to search by. | `ldap_mapped_attribute` | yes | `cn` | LDAP attribute to map to. | `ldap_search_time_limit` | no | 30 | LDAP search time limit in seconds. -| `ldap_ssl`* | no | 30 | LDAP TLS/SSL connection settings. +| `ldap_ssl`* | no | | LDAP TLS/SSL connection settings, see <>. +| `ad_guid_translation` | no | `auto` | Controls GUID binary conversion for Active Directory attributes. `auto` (default) converts when the LDAP search attribute equals `objectGUID` (case-insensitive). Use `always` to force conversion or `never` to disable it. | `ignore_missing` | no | false | Ignore errors when the source field is missing. | `ignore_failure` | no | false | Ignore all errors produced by the processor. |====== * Also see <> for a full description of the `ldap_ssl` options. +==== Server auto-discovery + +When `ldap_address` is omitted the processor attempts to discover controllers in +the following order: + +1. DNS SRV lookups for `_ldaps._tcp` (preferred) and `_ldap._tcp` using the + system's native DNS resolver. The processor queries multiple patterns: + * `_ldap._tcp.dc._msdcs.` (Active Directory domain controllers) + * `_ldap._tcp.` (standard domain lookup) + * If no domain is available, bare queries like `_ldap._tcp` which automatically + use the system's DNS search suffix configuration (works on Windows, Linux, and macOS) ++ +The domain used for DNS SRV lookups is determined from: + * The `ldap_domain` configuration option (highest priority) + * Operating-system domain metadata (for example session environment values or Windows domain-join settings) + * The host's fully qualified name when available + * Reverse DNS lookup of the local machine as a last resort ++ +NOTE: The processor uses Go's standard library `net.LookupSRV()` which leverages +the operating system's native DNS resolver. On Windows, this automatically reads +DNS servers and search suffixes from the Windows registry, making autodiscovery +work correctly even when running as a service without environment variables. SRV +records are automatically ordered by priority with weight-based randomization per +RFC 2782 to distribute load across available servers. +2. On Windows, the `LOGONSERVER` environment variable. The processor keeps the + hostname for TLS validation and may also try the resolved IP as a fallback. + +Every discovered server is tried sequentially until one responds. Likewise, if +`ldap_base_dn` is not supplied the client first queries the server's rootDSE for +`defaultNamingContext`/`namingContexts`, and if that fails, infers the DN from +the server's domain name (e.g. `dc=example,dc=com`). + If the searches are slow or you expect a high amount of different key attributes to be found, consider using a cache processor to speed processing: diff --git a/libbeat/processors/translate_ldap_attribute/guid.go b/libbeat/processors/translate_ldap_attribute/guid.go new file mode 100644 index 000000000000..a987a4c4294e --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/guid.go @@ -0,0 +1,102 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !requirefips + +package translate_ldap_attribute + +import ( + "encoding/hex" + "errors" + "fmt" + "strings" +) + +var ( + // errInvalidGUIDFormat is returned when the GUID format is invalid + errInvalidGUIDFormat = errors.New("invalid GUID format") +) + +// guidToBytes converts a GUID string in various formats to the binary format +// expected by Microsoft Active Directory. +// +// IMPORTANT: This conversion is ONLY for Microsoft Active Directory's objectGUID. +// Do NOT use for other LDAP implementations: +// - 389 Directory Server: Uses nsUniqueId (different format) +// - OpenLDAP and Other LDAP: Typically use RFC 4122 standard UUIDs +// +// Supported input formats: +// - {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} +// - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +// - xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +// +// The function handles the byte-order conversion required for Microsoft GUIDs: +// The first three components (Data1, Data2, Data3) are little-endian, +// while the remaining bytes are in network byte order. +// +// Example: +// +// Input: "{7fb125ee-ceaf-48ff-8385-32c516ab10ed}" +// Output: []byte{0xee, 0x25, 0xb1, 0x7f, 0xaf, 0xce, 0xff, 0x48, 0x83, 0x85, 0x32, 0xc5, 0x16, 0xab, 0x10, 0xed} +func guidToBytes(guid string) ([]byte, error) { + // Remove curly braces if present + guid = strings.Trim(guid, "{}") + + // Remove hyphens + guid = strings.ReplaceAll(guid, "-", "") + + // Validate length + if len(guid) != 32 { + return nil, fmt.Errorf("%w: expected 32 hex characters, got %d", errInvalidGUIDFormat, len(guid)) + } + + // Decode hex string + bytes, err := hex.DecodeString(guid) + if err != nil { + return nil, fmt.Errorf("%w: %v", errInvalidGUIDFormat, err) + } + + // Microsoft GUID format requires byte swapping for the first three components + // GUID structure: {Data1-Data2-Data3-Data4[8]} + // Data1: 4 bytes (little-endian) + // Data2: 2 bytes (little-endian) + // Data3: 2 bytes (little-endian) + // Data4: 8 bytes (big-endian/network order) + + // Swap Data1 (first 4 bytes) + bytes[0], bytes[1], bytes[2], bytes[3] = bytes[3], bytes[2], bytes[1], bytes[0] + + // Swap Data2 (next 2 bytes) + bytes[4], bytes[5] = bytes[5], bytes[4] + + // Swap Data3 (next 2 bytes) + bytes[6], bytes[7] = bytes[7], bytes[6] + + // Data4 remains in network byte order (no swap needed) + + return bytes, nil +} + +// escapeBinaryForLDAP escapes binary data for use in LDAP filters. +// Each byte is represented as \XX where XX is the hexadecimal value. +func escapeBinaryForLDAP(data []byte) string { + var sb strings.Builder + for _, b := range data { + fmt.Fprintf(&sb, "\\%02x", b) + } + return sb.String() +} diff --git a/libbeat/processors/translate_ldap_attribute/guid_test.go b/libbeat/processors/translate_ldap_attribute/guid_test.go new file mode 100644 index 000000000000..04cc2211acdf --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/guid_test.go @@ -0,0 +1,146 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !requirefips + +package translate_ldap_attribute + +import ( + "encoding/hex" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGUIDToBytes(t *testing.T) { + tests := []struct { + name string + input string + expected []byte + expectError bool + }{ + { + name: "GUID with curly braces and hyphens", + input: "{7fb125ee-ceaf-48ff-8385-32c516ab10ed}", + // Expected byte order after Microsoft GUID conversion: + // Original hex: 7fb125ee-ceaf-48ff-8385-32c516ab10ed + // After swap: ee25b17f-afce-ff48-8385-32c516ab10ed + expected: []byte{0xee, 0x25, 0xb1, 0x7f, 0xaf, 0xce, 0xff, 0x48, 0x83, 0x85, 0x32, 0xc5, 0x16, 0xab, 0x10, 0xed}, + expectError: false, + }, + { + name: "GUID with hyphens", + input: "7fb125ee-ceaf-48ff-8385-32c516ab10ed", + expected: []byte{0xee, 0x25, 0xb1, 0x7f, 0xaf, 0xce, 0xff, 0x48, 0x83, 0x85, 0x32, 0xc5, 0x16, 0xab, 0x10, 0xed}, + expectError: false, + }, + { + name: "GUID without hyphens", + input: "7fb125eeceaf48ff838532c516ab10ed", + expected: []byte{0xee, 0x25, 0xb1, 0x7f, 0xaf, 0xce, 0xff, 0x48, 0x83, 0x85, 0x32, 0xc5, 0x16, 0xab, 0x10, 0xed}, + expectError: false, + }, + { + name: "Another valid GUID", + input: "{a1b2c3d4-e5f6-0718-9293-a4b5c6d7e8f9}", + expected: []byte{0xd4, 0xc3, 0xb2, 0xa1, 0xf6, 0xe5, 0x18, 0x07, 0x92, 0x93, 0xa4, 0xb5, 0xc6, 0xd7, 0xe8, 0xf9}, + expectError: false, + }, + { + name: "Empty string", + input: "", + expected: nil, + expectError: true, + }, + { + name: "Invalid length", + input: "7fb125ee-ceaf-48ff-8385", + expected: nil, + expectError: true, + }, + { + name: "Invalid hex characters", + input: "7fb125ee-ceaf-48ff-8385-32c516ab10xz", + expected: nil, + expectError: true, + }, + { + name: "Too long", + input: "7fb125ee-ceaf-48ff-8385-32c516ab10ed-extra", + expected: nil, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := guidToBytes(tt.input) + + if tt.expectError { + assert.Error(t, err) + assert.Nil(t, result) + } else { + require.NoError(t, err) + assert.Equal(t, tt.expected, result, "Expected: %s, Got: %s", + hex.EncodeToString(tt.expected), + hex.EncodeToString(result)) + } + }) + } +} + +func TestEscapeBinaryForLDAP(t *testing.T) { + tests := []struct { + name string + input []byte + expected string + }{ + { + name: "Simple binary data", + input: []byte{0x7f, 0xb1, 0x25, 0xee}, + expected: "\\7f\\b1\\25\\ee", + }, + { + name: "GUID binary", + input: []byte{0xee, 0x25, 0xb1, 0x7f, 0xaf, 0xce, 0xff, 0x48, 0x83, 0x85, 0x32, 0xc5, 0x16, 0xab, 0x10, 0xed}, + expected: "\\ee\\25\\b1\\7f\\af\\ce\\ff\\48\\83\\85\\32\\c5\\16\\ab\\10\\ed", + }, + { + name: "Empty byte array", + input: []byte{}, + expected: "", + }, + { + name: "Single byte", + input: []byte{0x00}, + expected: "\\00", + }, + { + name: "High value bytes", + input: []byte{0xff, 0xfe, 0xfd}, + expected: "\\ff\\fe\\fd", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := escapeBinaryForLDAP(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/libbeat/processors/translate_ldap_attribute/ldap.go b/libbeat/processors/translate_ldap_attribute/ldap.go index 8eb2c8814f5a..862ed38cc121 100644 --- a/libbeat/processors/translate_ldap_attribute/ldap.go +++ b/libbeat/processors/translate_ldap_attribute/ldap.go @@ -22,6 +22,8 @@ package translate_ldap_attribute import ( "crypto/tls" "fmt" + "net" + "strings" "sync" "github.com/go-ldap/ldap/v3" @@ -36,6 +38,9 @@ type ldapClient struct { mu sync.Mutex conn *ldap.Conn + // Server metadata + isActiveDirectory bool + log *logp.Logger } @@ -50,7 +55,9 @@ type ldapConfig struct { tlsConfig *tls.Config } -// newLDAPClient initializes a new ldapClient with a single connection +// newLDAPClient initializes a new ldapClient with a single connection. +// If baseDN is empty, it will attempt to discover it via rootDSE or domain inference. +// It also detects whether the server is Active Directory. func newLDAPClient(config *ldapConfig, log *logp.Logger) (*ldapClient, error) { client := &ldapClient{ldapConfig: config, log: log} @@ -61,9 +68,222 @@ func newLDAPClient(config *ldapConfig, log *logp.Logger) (*ldapClient, error) { } client.conn = conn + // Discover base DN if not provided and detect AD + if err := client.initializeMetadata(); err != nil { + client.close() + return nil, fmt.Errorf("failed to initialize server metadata: %w", err) + } + return client, nil } +// initializeMetadata discovers base DN (if needed) and detects server type + +func (client *ldapClient) initializeMetadata() error { + client.log.Debug("querying rootDSE for server metadata") + + // Query rootDSE with relevant attributes + searchRequest := ldap.NewSearchRequest( + "", // Empty base DN = rootDSE + ldap.ScopeBaseObject, + ldap.NeverDerefAliases, + 0, // No size limit + client.searchTimeLimit, + false, + "(objectClass=*)", // Match everything + []string{ + "defaultNamingContext", + "namingContexts", + "rootDomainNamingContext", // AD-specific + "configurationNamingContext", // AD-specific + "schemaNamingContext", // AD-specific + "vendorName", + "vendorVersion", + }, + nil, + ) + + var result *ldap.SearchResult + err := client.withLockedConnection(func(conn *ldap.Conn) error { + var searchErr error + result, searchErr = conn.Search(searchRequest) + return searchErr + }) + if err != nil { + client.log.Debugw("rootDSE query failed", "error", err) + // If baseDN is already set, treat rootDSE failure as non-fatal + if client.baseDN != "" { + return nil + } + return client.inferBaseDNFromAddress() + } + + if len(result.Entries) == 0 { + client.log.Debug("rootDSE query returned no entries") + if client.baseDN != "" { + return nil + } + return client.inferBaseDNFromAddress() + } + + entry := result.Entries[0] + + // Detect Active Directory + // AD has rootDomainNamingContext, configurationNamingContext, and defaultNamingContext + hasRootDomain := len(entry.GetAttributeValues("rootDomainNamingContext")) > 0 + hasConfigContext := len(entry.GetAttributeValues("configurationNamingContext")) > 0 + hasDefaultContext := len(entry.GetAttributeValues("defaultNamingContext")) > 0 + + client.isActiveDirectory = hasRootDomain && hasConfigContext && hasDefaultContext + + if client.isActiveDirectory { + client.log.Info("detected Active Directory server") + } else { + vendorName := "" + if values := entry.GetAttributeValues("vendorName"); len(values) > 0 { + vendorName = values[0] + } + client.log.Infow("detected LDAP server", "vendor", vendorName) + } + + if client.baseDN != "" { + return nil + } + + // Prefer defaultNamingContext (Active Directory) + if values := entry.GetAttributeValues("defaultNamingContext"); len(values) > 0 { + client.baseDN = values[0] + client.log.Infow("discovered base DN via defaultNamingContext", "base_dn", client.baseDN) + return nil + } + + // Fallback to first namingContext + if values := entry.GetAttributeValues("namingContexts"); len(values) > 0 { + client.baseDN = values[0] + client.log.Infow("discovered base DN via namingContexts", "base_dn", client.baseDN) + return nil + } + + return client.inferBaseDNFromAddress() +} + +// inferBaseDNFromAddress infers base DN from the server address +func (client *ldapClient) inferBaseDNFromAddress() error { + client.log.Debugw("attempting to infer base DN from server address", "address", client.address) + + addr := client.address + lowerAddr := strings.ToLower(addr) + switch { + case strings.HasPrefix(lowerAddr, "ldap://"): + addr = addr[len("ldap://"):] + case strings.HasPrefix(lowerAddr, "ldaps://"): + addr = addr[len("ldaps://"):] + } + + var hostname string + h, _, err := net.SplitHostPort(addr) + if err != nil { + hostname = addr + if strings.Contains(hostname, ":") { + return fmt.Errorf("unable to parse hostname from address: %s", client.address) + } + } else { + hostname = h + } + + hostname = strings.TrimSuffix(hostname, ".") + hostname = strings.ToLower(hostname) + + if hostname == "" { + return fmt.Errorf("unable to extract hostname from address: %s", client.address) + } + + if net.ParseIP(hostname) != nil { + return fmt.Errorf("cannot infer base DN from IP address: %s", hostname) + } + + parts := strings.Split(hostname, ".") + if len(parts) < 2 { + return fmt.Errorf("hostname does not contain a domain: %s", hostname) + } + + var candidates [][]string + seen := make(map[string]struct{}) + addCandidate := func(parts []string) { + if len(parts) < 2 { + return + } + domain := strings.Join(parts, ".") + if _, ok := seen[domain]; ok { + return + } + seen[domain] = struct{}{} + candidates = append(candidates, append([]string(nil), parts...)) + } + + for i := 1; i < len(parts); i++ { + addCandidate(parts[i:]) + } + addCandidate(parts[len(parts)-2:]) + + if len(candidates) == 0 { + addCandidate(parts) + } + + validate := client.conn != nil + var fallbackDN string + for _, candidate := range candidates { + dn := domainPartsToDN(candidate) + if fallbackDN == "" { + fallbackDN = dn + } + if !validate { + continue + } + if err := client.validateBaseDN(dn); err != nil { + client.log.Debugw("base DN candidate validation failed", "candidate", dn, "error", err) + continue + } + client.baseDN = dn + client.log.Infow("inferred base DN from server domain", "base_dn", client.baseDN, "validated", true) + return nil + } + + if fallbackDN != "" { + client.baseDN = fallbackDN + client.log.Infow("inferred base DN from server domain", "base_dn", client.baseDN, "validated", false) + return nil + } + + return fmt.Errorf("unable to infer base DN from address: %s", client.address) +} + +func domainPartsToDN(parts []string) string { + dnParts := make([]string, 0, len(parts)) + for _, part := range parts { + dnParts = append(dnParts, fmt.Sprintf("dc=%s", part)) + } + return strings.Join(dnParts, ",") +} + +func (client *ldapClient) validateBaseDN(baseDN string) error { + return client.withLockedConnection(func(conn *ldap.Conn) error { + req := ldap.NewSearchRequest( + baseDN, + ldap.ScopeBaseObject, + ldap.NeverDerefAliases, + 1, + client.searchTimeLimit, + false, + "(objectClass=*)", + []string{"distinguishedName"}, + nil, + ) + _, err := conn.Search(req) + return err + }) +} + // dial establishes a new connection to the LDAP server func (client *ldapClient) dial() (*ldap.Conn, error) { client.log.Debugw("ldap client connecting") @@ -78,10 +298,21 @@ func (client *ldapClient) dial() (*ldap.Conn, error) { return nil, fmt.Errorf("failed to dial LDAP server: %w", err) } - if client.password != "" { - client.log.Debugw("ldap client bind") + // Bind with appropriate method + switch { + case client.password != "": + // Explicit credentials provided + client.log.Debugw("ldap client bind with provided credentials") err = conn.Bind(client.username, client.password) - } else { + case client.username == "" && client.password == "": + // No credentials: try Windows SSPI auth, fall back to unauthenticated + err = client.bindWithCurrentUser(conn) + if err != nil { + client.log.Debugw("Windows auth not available, falling back to unauthenticated bind", "error", err) + err = conn.UnauthenticatedBind("") + } + default: + // Username provided but no password: unauthenticated bind client.log.Debugw("ldap client unauthenticated bind") err = conn.UnauthenticatedBind(client.username) } @@ -94,30 +325,34 @@ func (client *ldapClient) dial() (*ldap.Conn, error) { return conn, nil } -// reconnect checks the connection's health and reconnects if necessary -func (client *ldapClient) connection() (*ldap.Conn, error) { +// connection checks the connection's health and reconnects if necessary +// withLockedConnection runs fn while holding the client mutex and ensuring the +// underlying LDAP connection is healthy before invoking the callback. +func (client *ldapClient) withLockedConnection(fn func(*ldap.Conn) error) error { client.mu.Lock() defer client.mu.Unlock() - // Check if the connection is still alive + if err := client.ensureConnectedLocked(); err != nil { + return err + } + return fn(client.conn) +} + +func (client *ldapClient) ensureConnectedLocked() error { if client.conn == nil || client.conn.IsClosing() { conn, err := client.dial() if err != nil { - return nil, err + return err } client.conn = conn } - return client.conn, nil + return nil } // findObjectBy searches for an object and returns its mapped values. -func (client *ldapClient) findObjectBy(searchBy string) ([]string, error) { - // Ensure the connection is alive or reconnect if necessary - conn, err := client.connection() - if err != nil { - return nil, fmt.Errorf("failed to reconnect: %w", err) - } +func (client *ldapClient) findObjectBy(searchBy string) ([]string, error) { + var result *ldap.SearchResult // Format the filter and perform the search filter := fmt.Sprintf("(%s=%s)", client.searchAttr, searchBy) searchRequest := ldap.NewSearchRequest( @@ -126,8 +361,12 @@ func (client *ldapClient) findObjectBy(searchBy string) ([]string, error) { filter, []string{client.mappedAttr}, nil, ) - // Execute search - result, err := conn.Search(searchRequest) + // Execute search while holding the connection lock to avoid concurrent usage of *ldap.Conn + err := client.withLockedConnection(func(conn *ldap.Conn) error { + var searchErr error + result, searchErr = conn.Search(searchRequest) + return searchErr + }) if err != nil { return nil, fmt.Errorf("search failed: %w", err) } @@ -135,9 +374,9 @@ func (client *ldapClient) findObjectBy(searchBy string) ([]string, error) { return nil, fmt.Errorf("no entries found for search attribute %s", searchBy) } - // Retrieve the CN attribute - cn := result.Entries[0].GetAttributeValues(client.mappedAttr) - return cn, nil + // Retrieve the mapped attribute values + values := result.Entries[0].GetAttributeValues(client.mappedAttr) + return values, nil } // close closes the LDAP connection diff --git a/libbeat/processors/translate_ldap_attribute/ldap_default.go b/libbeat/processors/translate_ldap_attribute/ldap_default.go new file mode 100644 index 000000000000..201759f146c5 --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/ldap_default.go @@ -0,0 +1,31 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !windows && !requirefips + +package translate_ldap_attribute + +import ( + "errors" + + "github.com/go-ldap/ldap/v3" +) + +// bindWithCurrentUser is a no-op on non-Windows platforms. +func (*ldapClient) bindWithCurrentUser(*ldap.Conn) error { + return errors.New("Windows SSPI authentication is only available on Windows") +} diff --git a/libbeat/processors/translate_ldap_attribute/ldap_test.go b/libbeat/processors/translate_ldap_attribute/ldap_test.go new file mode 100644 index 000000000000..97087f8b0458 --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/ldap_test.go @@ -0,0 +1,107 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !requirefips + +package translate_ldap_attribute + +import ( + "testing" + + "github.com/go-ldap/ldap/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrepareSearchFilter(t *testing.T) { + validGUID := "{7fb125ee-ceaf-48ff-8385-32c516ab10ed}" + guidBytes, _ := guidToBytes(validGUID) + expectedEscaped := escapeBinaryForLDAP(guidBytes) + + tests := []struct { + name string + ldapSearchAttr string + adGuidTranslation string + input string + expect string + expectErr bool + }{ + { + name: "Auto mode converts when attribute is objectGUID", + ldapSearchAttr: "objectGUID", + input: validGUID, + expect: expectedEscaped, + }, + { + name: "Auto mode is case-insensitive", + ldapSearchAttr: "objectguid", + input: validGUID, + expect: expectedEscaped, + }, + { + name: "Auto mode does not convert other attribute", + ldapSearchAttr: "uid", + input: validGUID, + expect: validGUID, + }, + { + name: "Explicit true converts even if attribute different", + ldapSearchAttr: "uid", + adGuidTranslation: guidTranslationAlways, + input: validGUID, + expect: expectedEscaped, + }, + { + name: "Explicit false never converts", + ldapSearchAttr: "objectGUID", + adGuidTranslation: guidTranslationNever, + input: validGUID, + expect: validGUID, + }, + { + name: "Invalid GUID with conversion attempt returns error", + ldapSearchAttr: "objectGUID", + input: "invalid-guid", + expectErr: true, + }, + { + name: "Escapes filter characters when not converting", + ldapSearchAttr: "uid", + input: "value*)(|(cn=*)", + expect: ldap.EscapeFilter("value*)(|(cn=*)"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &processor{ + config: config{ + LDAPSearchAttribute: tt.ldapSearchAttr, + ADGUIDTranslation: tt.adGuidTranslation, + }, + client: &ldapClient{}, + } + out, err := p.prepareSearchFilter(tt.input) + if tt.expectErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expect, out) + }) + } +} diff --git a/libbeat/processors/translate_ldap_attribute/ldap_windows.go b/libbeat/processors/translate_ldap_attribute/ldap_windows.go new file mode 100644 index 000000000000..53b946df165b --- /dev/null +++ b/libbeat/processors/translate_ldap_attribute/ldap_windows.go @@ -0,0 +1,64 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build windows && !requirefips + +package translate_ldap_attribute + +import ( + "fmt" + "net/url" + "strings" + + "github.com/go-ldap/ldap/v3" + "github.com/go-ldap/ldap/v3/gssapi" +) + +// bindWithCurrentUser performs GSSAPI bind using the current Windows user's credentials via SSPI. +func (client *ldapClient) bindWithCurrentUser(conn *ldap.Conn) error { + client.log.Info("using Windows SSPI authentication with current user credentials") + + // Create SSPI client using current process credentials + sspiClient, err := gssapi.NewSSPIClient() + if err != nil { + return fmt.Errorf("failed to create SSPI client: %w", err) + } + defer sspiClient.Close() + + // Extract hostname from LDAP address for SPN + parsedURL, err := url.Parse(client.address) + if err != nil { + return fmt.Errorf("failed to parse LDAP address: %w", err) + } + hostname := parsedURL.Hostname() + if hostname == "" { + return fmt.Errorf("could not extract hostname from address: %s", client.address) + } + + // Service Principal Name format: ldap/ + servicePrincipal := fmt.Sprintf("ldap/%s", strings.ToLower(hostname)) + client.log.Debugw("performing GSSAPI bind", "spn", servicePrincipal) + + // Perform GSSAPI bind + err = conn.GSSAPIBind(sspiClient, servicePrincipal, "") + if err != nil { + return fmt.Errorf("GSSAPI bind failed: %w", err) + } + + client.log.Info("GSSAPI bind successful") + return nil +} diff --git a/libbeat/processors/translate_ldap_attribute/translate_ldap_attribute.go b/libbeat/processors/translate_ldap_attribute/translate_ldap_attribute.go index 634e0f2f4252..d9c93c6462fc 100644 --- a/libbeat/processors/translate_ldap_attribute/translate_ldap_attribute.go +++ b/libbeat/processors/translate_ldap_attribute/translate_ldap_attribute.go @@ -22,6 +22,9 @@ package translate_ldap_attribute import ( "errors" "fmt" + "strings" + + "github.com/go-ldap/ldap/v3" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" @@ -52,13 +55,47 @@ func New(cfg *conf.C, log *logp.Logger) (beat.Processor, error) { if err := cfg.Unpack(&c); err != nil { return nil, fmt.Errorf("fail to unpack the translate_ldap_attribute configuration: %w", err) } + if err := c.validate(); err != nil { + return nil, fmt.Errorf("invalid translate_ldap_attribute configuration: %w", err) + } return newFromConfig(c, log) } func newFromConfig(c config, logger *logp.Logger) (*processor, error) { + p := &processor{config: c} + p.log = logger.Named(logName).With(logp.Stringer("processor", p)) + + client, err := newClient(c, p.log) + if err != nil { + return nil, err + } + + p.client = client + // Update config with the actual base DN used (may have been discovered) + p.LDAPBaseDN = client.baseDN + p.LDAPAddress = client.address + return p, nil +} + +// newClient creates a new LDAP client by discovering and connecting to available servers. +func newClient(c config, log *logp.Logger) (*ldapClient, error) { + // Auto-discover LDAP addresses if not provided + var addresses []string + if c.LDAPAddress != "" { + addresses = []string{c.LDAPAddress} + } else { + log.Info("LDAP address not configured, attempting auto-discovery") + discoveredAddresses, err := discoverLDAPAddress(c.LDAPDomain, log) + if err != nil { + return nil, fmt.Errorf("failed to auto-discover LDAP server: %w", err) + } + addresses = discoveredAddresses + log.Infow("discovered LDAP servers", "count", len(addresses), "addresses", addresses) + } + + // Prepare base LDAP config ldapConfig := &ldapConfig{ - address: c.LDAPAddress, baseDN: c.LDAPBaseDN, username: c.LDAPBindUser, password: c.LDAPBindPassword, @@ -67,20 +104,34 @@ func newFromConfig(c config, logger *logp.Logger) (*processor, error) { searchTimeLimit: c.LDAPSearchTimeLimit, } if c.LDAPTLS != nil { - tlsConfig, err := tlscommon.LoadTLSConfig(c.LDAPTLS, logger) + tlsConfig, err := tlscommon.LoadTLSConfig(c.LDAPTLS, log) if err != nil { return nil, fmt.Errorf("could not load provided LDAP TLS configuration: %w", err) } ldapConfig.tlsConfig = tlsConfig.ToConfig() } - p := &processor{config: c} - p.log = logger.Named(logName).With(logp.Stringer("processor", p)) - client, err := newLDAPClient(ldapConfig, p.log) - if err != nil { - return nil, err + + // Try each discovered address in order until one succeeds + var lastErr error + for i, address := range addresses { + log.Debugw("attempting to connect to LDAP server", "attempt", i+1, "total", len(addresses), "address", address) + ldapConfig.address = address + + // newLDAPClient handles connection, base DN discovery, and AD detection + client, err := newLDAPClient(ldapConfig, log) + if err != nil { + log.Debugw("failed to initialize LDAP client", "address", address, "error", err) + lastErr = err + continue + } + + // Successfully connected and initialized + log.Infow("successfully connected to LDAP server", "address", address, "base_dn", client.baseDN, "is_ad", client.isActiveDirectory) + return client, nil } - p.client = client - return p, nil + + // All addresses failed + return nil, fmt.Errorf("failed to connect to any discovered LDAP server (%d addresses tried): %w", len(addresses), lastErr) } func (p *processor) String() string { @@ -110,14 +161,19 @@ func (p *processor) translateLDAPAttr(event *beat.Event) error { return err } - guidString, ok := v.(string) + searchValue, ok := v.(string) if !ok { return errInvalidType } - p.log.Debugw("ldap search", "guid", guidString) - cn, err := p.client.findObjectBy(guidString) - p.log.Debugw("ldap result", "common_name", cn) + searchFilter, err := p.prepareSearchFilter(searchValue) + if err != nil { + return err + } + + p.log.Debugw("ldap search", "search_value", searchValue, "filter_value", searchFilter) + cn, err := p.client.findObjectBy(searchFilter) + p.log.Debugw("ldap result", "common_name", cn, "error", err) if err != nil { return err } @@ -130,6 +186,33 @@ func (p *processor) translateLDAPAttr(event *beat.Event) error { return err } +// prepareSearchFilter converts the search value to the appropriate format for LDAP queries. +// It applies GUID binary conversion when required based on the ADGUIDTranslation configuration +// and server type detection. +func (p *processor) prepareSearchFilter(searchValue string) (string, error) { + // Determine if GUID conversion should be applied + var shouldConvertGUID bool + switch p.ADGUIDTranslation { + case guidTranslationAlways: + shouldConvertGUID = true + case guidTranslationNever: + shouldConvertGUID = false + default: // auto + shouldConvertGUID = strings.EqualFold(p.LDAPSearchAttribute, "objectGUID") + } + + if !shouldConvertGUID { + return ldap.EscapeFilter(searchValue), nil + } + + guidBytes, err := guidToBytes(searchValue) + if err != nil { + return "", fmt.Errorf("failed to convert GUID: %w", err) + } + searchFilter := escapeBinaryForLDAP(guidBytes) + return searchFilter, nil +} + func (p *processor) Close() error { p.client.close() return nil