diff --git a/packages/elastic_agent/changelog.yml b/packages/elastic_agent/changelog.yml index 6b8ff9a1537..5f51b5203a1 100644 --- a/packages/elastic_agent/changelog.yml +++ b/packages/elastic_agent/changelog.yml @@ -1,5 +1,9 @@ # newer versions go on top - +- version: "2.6.1" + changes: + - description: Support new pipeline fields from beats logs + type: enhancement + link: https://github.com/elastic/integrations/pull/15280 - version: "2.6.0" changes: - description: Update version requirements due to the breaking change in metric names diff --git a/packages/elastic_agent/data_stream/auditbeat_logs/_dev/test/pipeline/test-queue-events-moved.json b/packages/elastic_agent/data_stream/auditbeat_logs/_dev/test/pipeline/test-queue-events-moved.json new file mode 100644 index 00000000000..cf55a5e3ceb --- /dev/null +++ b/packages/elastic_agent/data_stream/auditbeat_logs/_dev/test/pipeline/test-queue-events-moved.json @@ -0,0 +1,37 @@ +{ + "events": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "pct": { + "events": 0.10 + } + } + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.10 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/auditbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/auditbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json new file mode 100644 index 00000000000..a08a889a5b9 --- /dev/null +++ b/packages/elastic_agent/data_stream/auditbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json @@ -0,0 +1,33 @@ +{ + "expected": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": {} + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.1 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/auditbeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/auditbeat_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..c0b8a4d0d0b --- /dev/null +++ b/packages/elastic_agent/data_stream/auditbeat_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,17 @@ +--- +description: "Pipeline for adapting Elastic Agent fields" +processors: + - remove: + description: "Remove object that contained invalid data and can cause mapping conflicts" + field: monitoring.metrics.libbeat.pipeline.queue.filled.pct + if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map + ignore_missing: true + +on_failure: + - set: + field: error.message + value: "{{{_ingest.on_failure_message}}}" + - append: + field: event.kind + value: pipeline_error + allow_duplicates: false diff --git a/packages/elastic_agent/data_stream/auditbeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/auditbeat_logs/fields/fields.yml index 322de103b5f..63894b63a67 100644 --- a/packages/elastic_agent/data_stream/auditbeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/auditbeat_logs/fields/fields.yml @@ -70,8 +70,44 @@ - name: queue.max_events type: long metric_type: counter - description: Maximum number of events in a queue + description: Maximum number of events in a queue if it has one, otherwise zero. + - name: queue.max_bytes + type: long + metric_type: counter + description: Maximum number of bytes in a queue if it has one, otherwise zero. + - name: queue.filled.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.filled.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. - name: queue.filled.pct type: float metric_type: gauge description: Maximum number of events in a queue + - name: queue.added.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.added.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.consumed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.consumed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.removed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.removed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. diff --git a/packages/elastic_agent/data_stream/cloudbeat_logs/_dev/test/pipeline/test-queue-events-moved.json b/packages/elastic_agent/data_stream/cloudbeat_logs/_dev/test/pipeline/test-queue-events-moved.json new file mode 100644 index 00000000000..cf55a5e3ceb --- /dev/null +++ b/packages/elastic_agent/data_stream/cloudbeat_logs/_dev/test/pipeline/test-queue-events-moved.json @@ -0,0 +1,37 @@ +{ + "events": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "pct": { + "events": 0.10 + } + } + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.10 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/cloudbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/cloudbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json new file mode 100644 index 00000000000..a08a889a5b9 --- /dev/null +++ b/packages/elastic_agent/data_stream/cloudbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json @@ -0,0 +1,33 @@ +{ + "expected": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": {} + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.1 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/cloudbeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/cloudbeat_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..c0b8a4d0d0b --- /dev/null +++ b/packages/elastic_agent/data_stream/cloudbeat_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,17 @@ +--- +description: "Pipeline for adapting Elastic Agent fields" +processors: + - remove: + description: "Remove object that contained invalid data and can cause mapping conflicts" + field: monitoring.metrics.libbeat.pipeline.queue.filled.pct + if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map + ignore_missing: true + +on_failure: + - set: + field: error.message + value: "{{{_ingest.on_failure_message}}}" + - append: + field: event.kind + value: pipeline_error + allow_duplicates: false diff --git a/packages/elastic_agent/data_stream/cloudbeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/cloudbeat_logs/fields/fields.yml index abbb18cc82d..7f9b64ed30e 100644 --- a/packages/elastic_agent/data_stream/cloudbeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/cloudbeat_logs/fields/fields.yml @@ -71,8 +71,44 @@ - name: queue.max_events type: long metric_type: counter - description: Maximum number of events in a queue + description: Maximum number of events in a queue if it has one, otherwise zero. + - name: queue.max_bytes + type: long + metric_type: counter + description: Maximum number of bytes in a queue if it has one, otherwise zero. + - name: queue.filled.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.filled.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. - name: queue.filled.pct type: float metric_type: gauge description: Maximum number of events in a queue + - name: queue.added.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.added.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.consumed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.consumed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.removed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.removed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. diff --git a/packages/elastic_agent/data_stream/filebeat_logs/_dev/test/pipeline/test-queue-events-moved.json b/packages/elastic_agent/data_stream/filebeat_logs/_dev/test/pipeline/test-queue-events-moved.json new file mode 100644 index 00000000000..cf55a5e3ceb --- /dev/null +++ b/packages/elastic_agent/data_stream/filebeat_logs/_dev/test/pipeline/test-queue-events-moved.json @@ -0,0 +1,37 @@ +{ + "events": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "pct": { + "events": 0.10 + } + } + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.10 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/filebeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/filebeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json new file mode 100644 index 00000000000..a08a889a5b9 --- /dev/null +++ b/packages/elastic_agent/data_stream/filebeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json @@ -0,0 +1,33 @@ +{ + "expected": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": {} + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.1 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/filebeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/filebeat_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..c0b8a4d0d0b --- /dev/null +++ b/packages/elastic_agent/data_stream/filebeat_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,17 @@ +--- +description: "Pipeline for adapting Elastic Agent fields" +processors: + - remove: + description: "Remove object that contained invalid data and can cause mapping conflicts" + field: monitoring.metrics.libbeat.pipeline.queue.filled.pct + if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map + ignore_missing: true + +on_failure: + - set: + field: error.message + value: "{{{_ingest.on_failure_message}}}" + - append: + field: event.kind + value: pipeline_error + allow_duplicates: false diff --git a/packages/elastic_agent/data_stream/filebeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/filebeat_logs/fields/fields.yml index 322de103b5f..63894b63a67 100644 --- a/packages/elastic_agent/data_stream/filebeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/filebeat_logs/fields/fields.yml @@ -70,8 +70,44 @@ - name: queue.max_events type: long metric_type: counter - description: Maximum number of events in a queue + description: Maximum number of events in a queue if it has one, otherwise zero. + - name: queue.max_bytes + type: long + metric_type: counter + description: Maximum number of bytes in a queue if it has one, otherwise zero. + - name: queue.filled.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.filled.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. - name: queue.filled.pct type: float metric_type: gauge description: Maximum number of events in a queue + - name: queue.added.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.added.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.consumed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.consumed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.removed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.removed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. diff --git a/packages/elastic_agent/data_stream/heartbeat_logs/_dev/test/pipeline/test-queue-events-moved.json b/packages/elastic_agent/data_stream/heartbeat_logs/_dev/test/pipeline/test-queue-events-moved.json new file mode 100644 index 00000000000..cf55a5e3ceb --- /dev/null +++ b/packages/elastic_agent/data_stream/heartbeat_logs/_dev/test/pipeline/test-queue-events-moved.json @@ -0,0 +1,37 @@ +{ + "events": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "pct": { + "events": 0.10 + } + } + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.10 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/heartbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/heartbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json new file mode 100644 index 00000000000..a08a889a5b9 --- /dev/null +++ b/packages/elastic_agent/data_stream/heartbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json @@ -0,0 +1,33 @@ +{ + "expected": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": {} + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.1 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/heartbeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/heartbeat_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..c0b8a4d0d0b --- /dev/null +++ b/packages/elastic_agent/data_stream/heartbeat_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,17 @@ +--- +description: "Pipeline for adapting Elastic Agent fields" +processors: + - remove: + description: "Remove object that contained invalid data and can cause mapping conflicts" + field: monitoring.metrics.libbeat.pipeline.queue.filled.pct + if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map + ignore_missing: true + +on_failure: + - set: + field: error.message + value: "{{{_ingest.on_failure_message}}}" + - append: + field: event.kind + value: pipeline_error + allow_duplicates: false diff --git a/packages/elastic_agent/data_stream/heartbeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/heartbeat_logs/fields/fields.yml index 8d02cf66954..8308aeae66c 100644 --- a/packages/elastic_agent/data_stream/heartbeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/heartbeat_logs/fields/fields.yml @@ -73,8 +73,44 @@ - name: queue.max_events type: long metric_type: counter - description: Maximum number of events in a queue + description: Maximum number of events in a queue if it has one, otherwise zero. + - name: queue.max_bytes + type: long + metric_type: counter + description: Maximum number of bytes in a queue if it has one, otherwise zero. + - name: queue.filled.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.filled.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. - name: queue.filled.pct type: float metric_type: gauge description: Maximum number of events in a queue + - name: queue.added.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.added.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.consumed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.consumed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.removed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.removed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. diff --git a/packages/elastic_agent/data_stream/metricbeat_logs/_dev/test/pipeline/test-queue-events-moved.json b/packages/elastic_agent/data_stream/metricbeat_logs/_dev/test/pipeline/test-queue-events-moved.json new file mode 100644 index 00000000000..cf55a5e3ceb --- /dev/null +++ b/packages/elastic_agent/data_stream/metricbeat_logs/_dev/test/pipeline/test-queue-events-moved.json @@ -0,0 +1,37 @@ +{ + "events": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "pct": { + "events": 0.10 + } + } + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.10 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/metricbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/metricbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json new file mode 100644 index 00000000000..a08a889a5b9 --- /dev/null +++ b/packages/elastic_agent/data_stream/metricbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json @@ -0,0 +1,33 @@ +{ + "expected": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": {} + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.1 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/metricbeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/metricbeat_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..c0b8a4d0d0b --- /dev/null +++ b/packages/elastic_agent/data_stream/metricbeat_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,17 @@ +--- +description: "Pipeline for adapting Elastic Agent fields" +processors: + - remove: + description: "Remove object that contained invalid data and can cause mapping conflicts" + field: monitoring.metrics.libbeat.pipeline.queue.filled.pct + if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map + ignore_missing: true + +on_failure: + - set: + field: error.message + value: "{{{_ingest.on_failure_message}}}" + - append: + field: event.kind + value: pipeline_error + allow_duplicates: false diff --git a/packages/elastic_agent/data_stream/metricbeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/metricbeat_logs/fields/fields.yml index 322de103b5f..63894b63a67 100644 --- a/packages/elastic_agent/data_stream/metricbeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/metricbeat_logs/fields/fields.yml @@ -70,8 +70,44 @@ - name: queue.max_events type: long metric_type: counter - description: Maximum number of events in a queue + description: Maximum number of events in a queue if it has one, otherwise zero. + - name: queue.max_bytes + type: long + metric_type: counter + description: Maximum number of bytes in a queue if it has one, otherwise zero. + - name: queue.filled.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.filled.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. - name: queue.filled.pct type: float metric_type: gauge description: Maximum number of events in a queue + - name: queue.added.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.added.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.consumed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.consumed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.removed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.removed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. diff --git a/packages/elastic_agent/data_stream/osquerybeat_logs/_dev/test/pipeline/test-queue-events-moved.json b/packages/elastic_agent/data_stream/osquerybeat_logs/_dev/test/pipeline/test-queue-events-moved.json new file mode 100644 index 00000000000..cf55a5e3ceb --- /dev/null +++ b/packages/elastic_agent/data_stream/osquerybeat_logs/_dev/test/pipeline/test-queue-events-moved.json @@ -0,0 +1,37 @@ +{ + "events": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "pct": { + "events": 0.10 + } + } + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.10 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/osquerybeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/osquerybeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json new file mode 100644 index 00000000000..a08a889a5b9 --- /dev/null +++ b/packages/elastic_agent/data_stream/osquerybeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json @@ -0,0 +1,33 @@ +{ + "expected": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": {} + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.1 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/osquerybeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/osquerybeat_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..c0b8a4d0d0b --- /dev/null +++ b/packages/elastic_agent/data_stream/osquerybeat_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,17 @@ +--- +description: "Pipeline for adapting Elastic Agent fields" +processors: + - remove: + description: "Remove object that contained invalid data and can cause mapping conflicts" + field: monitoring.metrics.libbeat.pipeline.queue.filled.pct + if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map + ignore_missing: true + +on_failure: + - set: + field: error.message + value: "{{{_ingest.on_failure_message}}}" + - append: + field: event.kind + value: pipeline_error + allow_duplicates: false diff --git a/packages/elastic_agent/data_stream/osquerybeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/osquerybeat_logs/fields/fields.yml index 322de103b5f..63894b63a67 100644 --- a/packages/elastic_agent/data_stream/osquerybeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/osquerybeat_logs/fields/fields.yml @@ -70,8 +70,44 @@ - name: queue.max_events type: long metric_type: counter - description: Maximum number of events in a queue + description: Maximum number of events in a queue if it has one, otherwise zero. + - name: queue.max_bytes + type: long + metric_type: counter + description: Maximum number of bytes in a queue if it has one, otherwise zero. + - name: queue.filled.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.filled.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. - name: queue.filled.pct type: float metric_type: gauge description: Maximum number of events in a queue + - name: queue.added.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.added.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.consumed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.consumed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.removed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.removed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. diff --git a/packages/elastic_agent/data_stream/packetbeat_logs/_dev/test/pipeline/test-queue-events-moved.json b/packages/elastic_agent/data_stream/packetbeat_logs/_dev/test/pipeline/test-queue-events-moved.json new file mode 100644 index 00000000000..cf55a5e3ceb --- /dev/null +++ b/packages/elastic_agent/data_stream/packetbeat_logs/_dev/test/pipeline/test-queue-events-moved.json @@ -0,0 +1,37 @@ +{ + "events": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "pct": { + "events": 0.10 + } + } + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.10 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/packetbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json b/packages/elastic_agent/data_stream/packetbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json new file mode 100644 index 00000000000..a08a889a5b9 --- /dev/null +++ b/packages/elastic_agent/data_stream/packetbeat_logs/_dev/test/pipeline/test-queue-events-moved.json-expected.json @@ -0,0 +1,33 @@ +{ + "expected": [ + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": {} + } + } + } + } + } + }, + { + "monitoring": { + "metrics": { + "libbeat": { + "pipeline": { + "queue": { + "filled": { + "events": 42, + "pct": 0.1 + } + } + } + } + } + } + } + ] +} diff --git a/packages/elastic_agent/data_stream/packetbeat_logs/elasticsearch/ingest_pipeline/default.yml b/packages/elastic_agent/data_stream/packetbeat_logs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..c0b8a4d0d0b --- /dev/null +++ b/packages/elastic_agent/data_stream/packetbeat_logs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,17 @@ +--- +description: "Pipeline for adapting Elastic Agent fields" +processors: + - remove: + description: "Remove object that contained invalid data and can cause mapping conflicts" + field: monitoring.metrics.libbeat.pipeline.queue.filled.pct + if: ctx.monitoring?.metrics?.libbeat?.pipeline?.queue?.filled?.pct instanceof Map + ignore_missing: true + +on_failure: + - set: + field: error.message + value: "{{{_ingest.on_failure_message}}}" + - append: + field: event.kind + value: pipeline_error + allow_duplicates: false diff --git a/packages/elastic_agent/data_stream/packetbeat_logs/fields/fields.yml b/packages/elastic_agent/data_stream/packetbeat_logs/fields/fields.yml index 322de103b5f..63894b63a67 100644 --- a/packages/elastic_agent/data_stream/packetbeat_logs/fields/fields.yml +++ b/packages/elastic_agent/data_stream/packetbeat_logs/fields/fields.yml @@ -70,8 +70,44 @@ - name: queue.max_events type: long metric_type: counter - description: Maximum number of events in a queue + description: Maximum number of events in a queue if it has one, otherwise zero. + - name: queue.max_bytes + type: long + metric_type: counter + description: Maximum number of bytes in a queue if it has one, otherwise zero. + - name: queue.filled.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.filled.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. - name: queue.filled.pct type: float metric_type: gauge description: Maximum number of events in a queue + - name: queue.added.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.added.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.consumed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.consumed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. + - name: queue.removed.events + type: long + metric_type: gauge + description: Number of events currently stored in the queue. + - name: queue.removed.bytes + type: long + metric_type: gauge + description: Number of bytes currently in the queue. diff --git a/packages/elastic_agent/manifest.yml b/packages/elastic_agent/manifest.yml index 562c72045b0..e16d13da03f 100644 --- a/packages/elastic_agent/manifest.yml +++ b/packages/elastic_agent/manifest.yml @@ -1,13 +1,13 @@ name: elastic_agent title: Elastic Agent -version: 2.6.0 +version: 2.6.1 description: Collect logs and metrics from Elastic Agents. type: integration format_version: 3.1.4 categories: ["elastic_stack"] conditions: kibana: - version: "^8.15.0 || ^9.0.0" + version: "^8.11.2 || ^9.0.0" elastic: subscription: basic owner: