Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

filters/openpolicyagent: Add a sample benchmark to measure latency percentiles #3442

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
+86 −12
Draft

filters/openpolicyagent: Add a sample benchmark to measure latency percentiles #3442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 80 additions & 12 deletions filters/openpolicyagent/opaauthorizerequest/benchmark_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package opaauthorizerequest
import (
_ "embed"
"fmt"
"github.com/benburkert/pbench"
"github.com/golang-jwt/jwt/v4"
opasdktest "github.com/open-policy-agent/opa/sdk/test"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -252,21 +253,20 @@ func BenchmarkJwtValidation(b *testing.B) {
signedToken, err := token.SignedString(key)
require.NoError(b, err, "Failed to sign token")

ctx := &filtertest.Context{
FStateBag: map[string]interface{}{},
FResponse: &http.Response{},
FRequest: &http.Request{
Header: map[string][]string{
"Authorization": {fmt.Sprintf("Bearer %s", signedToken)},
},
URL: reqUrl,
},
FMetrics: &metricstest.MockMetrics{},
}

b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
ctx := &filtertest.Context{
FStateBag: map[string]interface{}{},
FResponse: &http.Response{},
FRequest: &http.Request{
Header: map[string][]string{
"Authorization": {fmt.Sprintf("Bearer %s", signedToken)},
},
URL: reqUrl,
},
FMetrics: &metricstest.MockMetrics{},
}
f.Request(ctx)
assert.False(b, ctx.FServed)
}
Expand Down Expand Up @@ -329,6 +329,74 @@ func BenchmarkMinimalPolicyBundle(b *testing.B) {
})
}

// BenchmarkWithPercentiles is an example benchmark that demonstrates how to measure
// and report latency percentiles in OPA filter benchmark tests.
//
// It uses a minimal policy with decision logging enabled and reports
// p50, p95, p99, and p999 latency percentiles to provide insights into
// the distribution of request latencies, especially the impact of
// decision logging.
//
// To run this benchmark, use the following command:
// go test -bench=^BenchmarkWithPercentiles$ -benchmem ./filters/openpolicyagent/opaauthorizerequest
func BenchmarkWithPercentiles(b *testing.B) {
opaControlPlane := opasdktest.MustNewServer(
opasdktest.MockBundle(testBundleEndpoint, map[string]string{
"main.rego": `
package envoy.authz

default allow = false

allow {
input.parsed_path = [ "allow" ]
}
`,
}),
)
defer opaControlPlane.Stop()

decisionLogsConsumer := newDecisionConsumer()
defer decisionLogsConsumer.Close()

filterOpts := FilterOptions{
OpaControlPlaneUrl: opaControlPlane.URL(),
DecisionConsumerUrl: decisionLogsConsumer.URL,
DecisionPath: testDecisionPath,
BundleName: testBundleName,
DecisionLogging: true,
ContextExtensions: "",
}
f, err := createOpaFilter(filterOpts)
require.NoError(b, err)

reqUrl, err := url.Parse("http://opa-authorized.test/allow")
require.NoError(b, err)

ctx := &filtertest.Context{
FStateBag: map[string]interface{}{},
FResponse: &http.Response{},
FRequest: &http.Request{
URL: reqUrl,
},
FMetrics: &metricstest.MockMetrics{},
}

pb := pbench.New(b)
pb.ReportPercentile(0.5)
pb.ReportPercentile(0.95)
pb.ReportPercentile(0.99)
pb.ReportPercentile(0.999)

pb.Run("minimal-with-decision-logs", func(b *pbench.B) {
b.RunParallel(func(pb *pbench.PB) {
for pb.Next() {
f.Request(ctx)
assert.False(b, ctx.FServed)
}
})
})
}

func newDecisionConsumer() *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
Expand Down
2 changes: 2 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/abbot/go-http-auth v0.4.0
github.com/andybalholm/brotli v1.1.1
github.com/aryszka/jobqueue v0.0.3
github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/cespare/xxhash/v2 v2.3.0
github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9
Expand Down Expand Up @@ -94,6 +95,7 @@ require (
github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
github.com/go-logr/logr v1.4.2 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ
github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
github.com/aryszka/jobqueue v0.0.3 h1:O5YbgzQCjRomudwnDTY5BrHUNJhvPHQHq7GfGpE+ybs=
github.com/aryszka/jobqueue v0.0.3/go.mod h1:SdxqI6HZ4E1Lss94tey5OfjcAu3bdCDWS1AQzzIN4m4=
github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef h1:+7ZJvJGiV4hUBdjhEDhfGdjBCOmhVi0YQ5n+6g/ei+k=
Copy link
Collaborator Author

@mefarazath mefarazath Mar 18, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AlexanderYastrebov based on comments in PR, i guess we are not okay with using this dependency.

Is it fine if we copy over the function/s used from library to our codebase?

Copy link
Member

@AlexanderYastrebov AlexanderYastrebov Mar 20, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After reading the code of the library I am not sure it does the right thing, e.g. I don't see how it accounts for warmup benchmark runs. Lets first understand what we want to achieve.

The standard way to compare benchmark results is to use https://pkg.go.dev/golang.org/x/perf/cmd/benchstat

The table then compares the two input files for each benchmark. It shows the median and 95% confidence interval summaries for each benchmark before and after the change, and an A/B comparison under "vs base".

Checking help for benchstat:

$ benchstat
Usage: benchstat [flags] inputs...

benchstat computes statistical summaries and A/B comparisons of Go
benchmarks. It shows benchmark medians in a table with a row for each
benchmark and a column for each input file. If there is more than one
input file, it also shows A/B comparisons between the files. If a
difference is likely to be noise, it shows "~".

For details, see https://pkg.go.dev/golang.org/x/perf/cmd/benchstat.
  -alpha α
    	consider change significant if p < α (default 0.05)
  -col projection
    	split results into columns by distinct values of projection (default ".file")
  -confidence level
    	confidence level for ranges (default 0.95)
  -filter query
    	use only benchmarks matching benchfilter query (default "*")
  -format format
    	print results in format:
    	  text - plain text
    	  csv  - comma-separated values (warnings will be written to stderr)
    	 (default "text")
  -ignore keys
    	ignore variations in keys
  -row projection
    	split results into rows by distinct values of projection (default ".fullname")
  -table projection
    	split results into tables by distinct values of projection (default ".config")

it looks like we can change -confidence if we need other than p95.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also if you want to compare benchmark results for two commits you may use this script golang/go#63233 (comment)

github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef/go.mod h1:hrhDSsc41bBqGejYXbvMh6qexfcC2vXjodP5gufwWyI=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
Expand Down Expand Up @@ -153,6 +155,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 h1:Vh7rylVZRZCj6W41lRlP17xPk4Nq260H4Xo/DDYmEZk=
github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
Expand Down
Loading