Skip to content

Commit

Permalink
tests: add EXPECT_METRICS_EXIST to integration tests (#771)
Browse files Browse the repository at this point in the history 
Integration tests sometimes don't care about the exact set of metrics
that are being sent (particularly around things like datastore metrics);
what is really interesting is that the framework is detected, the
transaction is named correctly, and that there aren't errors. This adds
a new `EXPECT_METRICS_EXIST` section which is simply a newline
separated list of metric names that will be searched for: if any don't
exist, the test will fail.

For example:

```php
<?php

/*EXPECT_METRICS_EXIST
Supportability/framework/WordPress/detected
*/
?>
```

This asserts that the test will generate
`Supportability/framework/WordPress/detected` metric, but doesn't assert
anything else about the metric table.

---------

Co-authored-by: Adam Harvey <[email protected]>
  • Loading branch information
@lavarou @LawnGnome
lavarou and LawnGnome committed Nov 27, 2023
1 parent 407e670 commit 0186678
Show file tree
Hide file tree
Showing 8 changed files with 52 additions and 79 deletions.
5 changes: 5 additions & 0 deletions src/newrelic/integration/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ var (
"EXPECT_SPAN_EVENTS_LIKE": parseSpanEventsLike,
"EXPECT_LOG_EVENTS": parseLogEvents,
"EXPECT_METRICS": parseMetrics,
"EXPECT_METRICS_EXIST": parseMetricsExist,
"EXPECT": parseExpect,
"EXPECT_REGEX": parseExpectRegex,
"EXPECT_SCRUBBED": parseExpectScrubbed,
Expand Down Expand Up @@ -224,6 +225,10 @@ func parseMetrics(test *Test, content []byte) error {
test.metrics = content
return nil
}
func parseMetricsExist(test *Test, content []byte) error {
test.metricsExist = content
return nil
}
func parseSlowSQLs(test *Test, content []byte) error {
test.slowSQLs = content
return nil
Expand Down
10 changes: 10 additions & 0 deletions src/newrelic/integration/test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ type Test struct {
spanEventsLike []byte
logEvents []byte
metrics []byte
metricsExist []byte
slowSQLs []byte
tracedErrors []byte
txnTraces []byte
Expand Down Expand Up @@ -557,6 +558,15 @@ func (t *Test) Compare(harvest *newrelic.Harvest) {
return
}

if nil != t.metricsExist {
for _, name := range strings.Split(strings.TrimSpace(string(t.metricsExist)), "\n") {
name = strings.TrimSpace(name)
if !harvest.Metrics.Has(name) {
t.Fail(fmt.Errorf("metric does not exist: %s\n\nactual metric table: %s", name, harvest.Metrics.DebugJSON()))
}
}
}

// if we expect a harvest and these is not, then we run our tests as per normal
t.compareResponseHeaders()

Expand Down
7 changes: 7 additions & 0 deletions src/newrelic/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,13 @@ func (mt *MetricTable) Empty() bool {
return 0 == mt.count
}

// Has returns true if the given metric exists in the metric table (regardless
// of scope).
func (mt *MetricTable) Has(name string) bool {
_, ok := mt.metrics[name]
return ok
}

// Data marshals the collection to JSON according to the schema expected
// by the collector.
func (mt *MetricTable) Data(id AgentRunID, harvestStart time.Time) ([]byte,
Expand Down
18 changes: 18 additions & 0 deletions src/newrelic/metrics_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,3 +301,21 @@ func BenchmarkMetricTableCollectorJSON(b *testing.B) {
mt.CollectorJSON(AgentRunID("12345"), now)
}
}

func TestMetricTableHas(t *testing.T) {
mt := NewMetricTable(20, start)
mt.AddCount("foo", "", 1, 0)
mt.AddCount("bar", "quux", 1, 0)

if mt.Has("quux") {
t.Fatal("non-existent metric is reported as existing")
}

if !mt.Has("foo") {
t.Fatal("unscoped metric is reported as missing")
}

if !mt.Has("bar") {
t.Fatal("scoped metric is reported as missing")
}
}
22 changes: 3 additions & 19 deletions tests/integration/logging/analog/test_supportability_metric.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,9 @@
newrelic.application_logging.enabled = true
*/

/*EXPECT_METRICS
[
"?? agent run id",
"?? timeframe start",
"?? timeframe stop",
[
[{"name":"DurationByCaller/Unknown/Unknown/Unknown/Unknown/all"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"DurationByCaller/Unknown/Unknown/Unknown/Unknown/allOther"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransaction/all"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransaction/php__FILE__"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransactionTotalTime"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransactionTotalTime/php__FILE__"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/PHP/Analog/disabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/library/Analog/detected"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/Forwarding/PHP/enabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/Metrics/PHP/enabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/LocalDecorating/PHP/disabled"}, [1, "??", "??", "??", "??", "??"]]
]
]
/*EXPECT_METRICS_EXIST
Supportability/library/Analog/detected
Supportability/Logging/PHP/Analog/disabled
*/


Expand Down
23 changes: 3 additions & 20 deletions tests/integration/logging/cakephp-log/test_supportability_metric.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,9 @@
newrelic.application_logging.enabled = true
*/

/*EXPECT_METRICS
[
"?? agent run id",
"?? timeframe start",
"?? timeframe stop",
[
[{"name":"DurationByCaller/Unknown/Unknown/Unknown/Unknown/all"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"DurationByCaller/Unknown/Unknown/Unknown/Unknown/allOther"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransaction/all"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransaction/php__FILE__"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransactionTotalTime"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransactionTotalTime/php__FILE__"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/PHP/cakephp-log/disabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/library/cakephp-log/detected"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/Forwarding/PHP/enabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/Metrics/PHP/enabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/LocalDecorating/PHP/disabled"}, [1, "??", "??", "??", "??", "??"]]
]
]
/*EXPECT_METRICS_EXIST
Supportability/library/cakephp-log/detected
Supportability/Logging/PHP/cakephp-log/disabled
*/


require_once(realpath(dirname(__FILE__)) . '/vendor/cakephp/log/Log.php');
23 changes: 3 additions & 20 deletions tests/integration/logging/consolidation-log/test_supportability_metric.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,9 @@
newrelic.application_logging.enabled = true
*/

/*EXPECT_METRICS
[
"?? agent run id",
"?? timeframe start",
"?? timeframe stop",
[
[{"name":"DurationByCaller/Unknown/Unknown/Unknown/Unknown/all"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"DurationByCaller/Unknown/Unknown/Unknown/Unknown/allOther"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransaction/all"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransaction/php__FILE__"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransactionTotalTime"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransactionTotalTime/php__FILE__"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/PHP/Consolidation/Log/disabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/library/Consolidation/Log/detected"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/Forwarding/PHP/enabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/Metrics/PHP/enabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/LocalDecorating/PHP/disabled"}, [1, "??", "??", "??", "??", "??"]]
]
]
/*EXPECT_METRICS_EXIST
Supportability/library/Consolidation/Log/detected
Supportability/Logging/PHP/Consolidation/Log/disabled
*/


require_once(realpath(dirname(__FILE__)) . '/vendor/consolidation/log/src/Logger.php');
23 changes: 3 additions & 20 deletions tests/integration/logging/laminas-log/test_supportability_metric.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,9 @@
newrelic.application_logging.enabled = true
*/

/*EXPECT_METRICS
[
"?? agent run id",
"?? timeframe start",
"?? timeframe stop",
[
[{"name":"DurationByCaller/Unknown/Unknown/Unknown/Unknown/all"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"DurationByCaller/Unknown/Unknown/Unknown/Unknown/allOther"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransaction/all"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransaction/php__FILE__"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransactionTotalTime"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"OtherTransactionTotalTime/php__FILE__"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/PHP/laminas-log/disabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/library/laminas-log/detected"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/Forwarding/PHP/enabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/Metrics/PHP/enabled"}, [1, "??", "??", "??", "??", "??"]],
[{"name":"Supportability/Logging/LocalDecorating/PHP/disabled"}, [1, "??", "??", "??", "??", "??"]]
]
]
/*EXPECT_METRICS_EXIST
Supportability/library/laminas-log/detected
Supportability/Logging/PHP/laminas-log/disabled
*/


require_once(realpath(dirname(__FILE__)) . '/vendor/laminas/laminas-log/src/Logger.php');

0 comments on commit 0186678

Please sign in to comment.