Skip to content

Commit 2dcf1c9

Browse files
strantalisstrantalis
committed
feat(examples): add complex authorization benchmarks
Add new benchmark commands that exercise subject mapping evaluation with multiple attributes per resource and multiple resources per request: - benchmark-decision-complex: v2 API with configurable resources/attrs - benchmark-decision-v1-complex: v1 API with same capabilities These benchmarks show where optimization improvements shine by testing with 500 resources × 5 attributes = 2,500 attribute checks per request. Also updates CI workflow to run and report these benchmarks.
1 parent b7c396c commit 2dcf1c9

File tree

4 files changed

+295
-2
lines changed

4 files changed

+295
-2
lines changed

‎.github/workflows/checks.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,28 @@ jobs:
273273
echo "$OUTPUT";
274274
echo "EODECISIONV2"
275275
} >> "$GITHUB_ENV"
276+
- name: run complex decision benchmark tests
277+
run: |
278+
OUTPUT=$(./examples/examples benchmark-decision-complex --resources 500 --attrs 5)
279+
echo "--- Complex Decision Benchmark Output ---"
280+
echo "$OUTPUT"
281+
echo "$OUTPUT" >> "$GITHUB_STEP_SUMMARY"
282+
{
283+
echo "BENCHMARK_DECISION_COMPLEX_OUTPUT<<EODECISIONCOMPLEX";
284+
echo "$OUTPUT";
285+
echo "EODECISIONCOMPLEX"
286+
} >> "$GITHUB_ENV"
287+
- name: run complex decision v1 benchmark tests
288+
run: |
289+
OUTPUT=$(./examples/examples benchmark-decision-v1-complex --resources 500 --attrs 5)
290+
echo "--- Complex Decision v1 Benchmark Output ---"
291+
echo "$OUTPUT"
292+
echo "$OUTPUT" >> "$GITHUB_STEP_SUMMARY"
293+
{
294+
echo "BENCHMARK_DECISION_V1_COMPLEX_OUTPUT<<EODECISIONV1COMPLEX";
295+
echo "$OUTPUT";
296+
echo "EODECISIONV1COMPLEX"
297+
} >> "$GITHUB_ENV"
276298
- name: run tdf3 benchmark tests
277299
run: |
278300
OUTPUT=$(./examples/examples benchmark --count=5000 --concurrent=50)
@@ -322,6 +344,8 @@ jobs:
322344
323345
h2 "${BENCHMARK_DECISION_OUTPUT}" "Decision Benchmark"
324346
h2 "${BENCHMARK_DECISION_V2_OUTPUT}" "Decision Benchmark v2"
347+
h2 "${BENCHMARK_DECISION_COMPLEX_OUTPUT}" "Complex Decision Benchmark"
348+
h2 "${BENCHMARK_DECISION_V1_COMPLEX_OUTPUT}" "Complex Decision v1 Benchmark"
325349
h2 "${BENCHMARK_METRICS_OUTPUT}" "Standard Benchmark Metrics"
326350
h2 "${BENCHMARK_BULK_OUTPUT}" "Bulk Benchmark"
327351
h2 "${BENCHMARK_TDF3_OUTPUT}" "TDF3 Benchmark"

‎examples/cmd/benchmark_decision_complex.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
//nolint:forbidigo // We use Println here extensively because we are printing markdown.
2+
package cmd
3+
4+
import (
5+
"context"
6+
"fmt"
7+
"strconv"
8+
"time"
9+
10+
authzV2 "github.com/opentdf/platform/protocol/go/authorization/v2"
11+
"github.com/opentdf/platform/protocol/go/entity"
12+
"github.com/opentdf/platform/protocol/go/policy"
13+
"github.com/spf13/cobra"
14+
)
15+
16+
var (
17+
complexAttrCount int
18+
complexResourceCount int
19+
)
20+
21+
func init() {
22+
benchmarkCmd := &cobra.Command{
23+
Use: "benchmark-decision-complex",
24+
Short: "benchmark authorization with complex policy scenarios",
25+
Long: `A benchmark tool to measure authorization performance with complex policies.
26+
This benchmark exercises subject mapping evaluation with multiple attributes per resource
27+
and multiple resources per request, showing where optimization improvements shine.`,
28+
RunE: runDecisionBenchmarkComplex,
29+
}
30+
benchmarkCmd.Flags().IntVar(&complexAttrCount, "attrs", 5, "Number of attributes per resource") //nolint:mnd // default shown in help
31+
benchmarkCmd.Flags().IntVar(&complexResourceCount, "resources", 100, "Number of resources per request") //nolint:mnd // default shown in help
32+
ExamplesCmd.AddCommand(benchmarkCmd)
33+
}
34+
35+
func runDecisionBenchmarkComplex(_ *cobra.Command, _ []string) error {
36+
client, err := newSDK()
37+
if err != nil {
38+
return err
39+
}
40+
41+
// Available attribute values from fixtures that have subject mappings
42+
// These exercise different subject mapping conditions
43+
availableAttrs := []string{
44+
"https://example.com/attr/attr1/value/value1", // 4 subject mappings with complex AND/OR conditions
45+
"https://example.com/attr/attr1/value/value2", // 1 subject mapping
46+
"https://example.net/attr/attr1/value/value1", // 1 subject mapping with NOT_IN condition
47+
"https://example.com/attr/attr2/value/value1", // ALL_OF rule
48+
"https://example.com/attr/attr2/value/value2", // ALL_OF rule
49+
}
50+
51+
// Build resources with multiple attributes each
52+
var resources []*authzV2.Resource
53+
for i := 0; i < complexResourceCount; i++ {
54+
// Select attributes for this resource (cycle through available ones)
55+
var fqns []string
56+
for j := 0; j < complexAttrCount; j++ {
57+
fqns = append(fqns, availableAttrs[(i+j)%len(availableAttrs)])
58+
}
59+
60+
r := &authzV2.Resource{
61+
EphemeralId: "resource-" + strconv.Itoa(i),
62+
Resource: &authzV2.Resource_AttributeValues_{
63+
AttributeValues: &authzV2.Resource_AttributeValues{
64+
Fqns: fqns,
65+
},
66+
},
67+
}
68+
resources = append(resources, r)
69+
}
70+
71+
// Run the benchmark multiple times to get stable measurements
72+
const iterations = 5
73+
var totalTime time.Duration
74+
var lastApproved, lastDenied int
75+
76+
for iter := 0; iter < iterations; iter++ {
77+
start := time.Now()
78+
res, err := client.AuthorizationV2.GetDecisionMultiResource(context.Background(), &authzV2.GetDecisionMultiResourceRequest{
79+
Action: &policy.Action{
80+
Name: "read",
81+
},
82+
EntityIdentifier: &authzV2.EntityIdentifier{
83+
Identifier: &authzV2.EntityIdentifier_EntityChain{
84+
EntityChain: &entity.EntityChain{
85+
EphemeralId: "complex-benchmark-entity-chain",
86+
Entities: []*entity.Entity{
87+
{
88+
EphemeralId: "env-cli-client",
89+
EntityType: &entity.Entity_ClientId{ClientId: "opentdf-sdk"},
90+
Category: entity.Entity_CATEGORY_ENVIRONMENT,
91+
},
92+
{
93+
EphemeralId: "subject-sample-user",
94+
EntityType: &entity.Entity_UserName{UserName: "sample-user"},
95+
Category: entity.Entity_CATEGORY_SUBJECT,
96+
},
97+
},
98+
},
99+
},
100+
},
101+
Resources: resources,
102+
})
103+
elapsed := time.Since(start)
104+
totalTime += elapsed
105+
106+
if err != nil {
107+
return fmt.Errorf("iteration %d failed: %w", iter, err)
108+
}
109+
110+
lastApproved = 0
111+
lastDenied = 0
112+
for _, decision := range res.GetResourceDecisions() {
113+
if decision.GetDecision() == authzV2.Decision_DECISION_PERMIT {
114+
lastApproved++
115+
} else {
116+
lastDenied++
117+
}
118+
}
119+
}
120+
121+
avgTime := totalTime / iterations
122+
decisionsPerSecond := float64(complexResourceCount) / avgTime.Seconds()
123+
124+
// Print results
125+
fmt.Printf("# Complex Authorization Benchmark Results\n\n")
126+
fmt.Printf("## Configuration\n")
127+
fmt.Printf("| Parameter | Value |\n")
128+
fmt.Printf("|----------------------|------------------------|\n")
129+
fmt.Printf("| Resources per request | %d |\n", complexResourceCount)
130+
fmt.Printf("| Attributes per resource | %d |\n", complexAttrCount)
131+
fmt.Printf("| Total attribute checks | %d |\n", complexResourceCount*complexAttrCount)
132+
fmt.Printf("| Iterations | %d |\n", iterations)
133+
fmt.Printf("\n")
134+
fmt.Printf("## Results\n")
135+
fmt.Printf("| Metric | Value |\n")
136+
fmt.Printf("|-------------------------|------------------------|\n")
137+
fmt.Printf("| Approved Decisions | %d |\n", lastApproved)
138+
fmt.Printf("| Denied Decisions | %d |\n", lastDenied)
139+
fmt.Printf("| Average Request Time | %s |\n", avgTime)
140+
fmt.Printf("| Decisions/second | %.2f |\n", decisionsPerSecond)
141+
fmt.Printf("| Time per decision | %s |\n", time.Duration(int64(avgTime)/int64(complexResourceCount)))
142+
143+
return nil
144+
}

‎examples/cmd/benchmark_decision_v1_complex.go‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
//nolint:forbidigo // We use Println here extensively because we are printing markdown.
2+
package cmd
3+
4+
import (
5+
"context"
6+
"fmt"
7+
"time"
8+
9+
"github.com/opentdf/platform/protocol/go/authorization"
10+
"github.com/opentdf/platform/protocol/go/policy"
11+
"github.com/spf13/cobra"
12+
)
13+
14+
func init() {
15+
benchmarkCmd := &cobra.Command{
16+
Use: "benchmark-decision-v1-complex",
17+
Short: "benchmark authorization.GetDecisions with complex policy scenarios",
18+
Long: `A benchmark tool to measure authorization v1 performance with complex policies.
19+
This benchmark exercises subject mapping evaluation with multiple attributes per resource.`,
20+
RunE: runDecisionBenchmarkV1Complex,
21+
}
22+
benchmarkCmd.Flags().IntVar(&complexAttrCount, "attrs", 5, "Number of attributes per resource") //nolint:mnd // default shown in help
23+
benchmarkCmd.Flags().IntVar(&complexResourceCount, "resources", 100, "Number of resources per request") //nolint:mnd // default shown in help
24+
ExamplesCmd.AddCommand(benchmarkCmd)
25+
}
26+
27+
func runDecisionBenchmarkV1Complex(_ *cobra.Command, _ []string) error {
28+
client, err := newSDK()
29+
if err != nil {
30+
return err
31+
}
32+
33+
// Available attribute values from fixtures that have subject mappings
34+
availableAttrs := []string{
35+
"https://example.com/attr/attr1/value/value1",
36+
"https://example.com/attr/attr1/value/value2",
37+
"https://example.net/attr/attr1/value/value1",
38+
"https://example.com/attr/attr2/value/value1",
39+
"https://example.com/attr/attr2/value/value2",
40+
}
41+
42+
// Build resource attributes with multiple attribute values each
43+
var ras []*authorization.ResourceAttribute
44+
for i := 0; i < complexResourceCount; i++ {
45+
var fqns []string
46+
for j := 0; j < complexAttrCount; j++ {
47+
fqns = append(fqns, availableAttrs[(i+j)%len(availableAttrs)])
48+
}
49+
ras = append(ras, &authorization.ResourceAttribute{AttributeValueFqns: fqns})
50+
}
51+
52+
// Run benchmark iterations
53+
const iterations = 5
54+
var totalTime time.Duration
55+
var lastApproved, lastDenied int
56+
57+
for iter := 0; iter < iterations; iter++ {
58+
start := time.Now()
59+
res, err := client.Authorization.GetDecisions(context.Background(), &authorization.GetDecisionsRequest{
60+
DecisionRequests: []*authorization.DecisionRequest{
61+
{
62+
Actions: []*policy.Action{{Value: &policy.Action_Standard{
63+
Standard: policy.Action_STANDARD_ACTION_DECRYPT,
64+
}}},
65+
EntityChains: []*authorization.EntityChain{
66+
{
67+
Id: "complex-v1-entity-chain",
68+
Entities: []*authorization.Entity{
69+
{
70+
Id: "env-cli-client",
71+
EntityType: &authorization.Entity_ClientId{ClientId: "opentdf-sdk"},
72+
Category: authorization.Entity_CATEGORY_ENVIRONMENT,
73+
},
74+
{
75+
Id: "subject-sample-user",
76+
EntityType: &authorization.Entity_UserName{UserName: "sample-user"},
77+
Category: authorization.Entity_CATEGORY_SUBJECT,
78+
},
79+
},
80+
},
81+
},
82+
ResourceAttributes: ras,
83+
},
84+
},
85+
})
86+
elapsed := time.Since(start)
87+
totalTime += elapsed
88+
89+
if err != nil {
90+
return fmt.Errorf("iteration %d failed: %w", iter, err)
91+
}
92+
93+
lastApproved = 0
94+
lastDenied = 0
95+
for _, dr := range res.GetDecisionResponses() {
96+
if dr.GetDecision() == authorization.DecisionResponse_DECISION_PERMIT {
97+
lastApproved++
98+
} else {
99+
lastDenied++
100+
}
101+
}
102+
}
103+
104+
avgTime := totalTime / iterations
105+
decisionsPerSecond := float64(complexResourceCount) / avgTime.Seconds()
106+
107+
// Print results
108+
fmt.Printf("# Complex Authorization v1 Benchmark Results\n\n")
109+
fmt.Printf("## Configuration\n")
110+
fmt.Printf("| Parameter | Value |\n")
111+
fmt.Printf("|------------------------|------------------------|\n")
112+
fmt.Printf("| Resources per request | %d |\n", complexResourceCount)
113+
fmt.Printf("| Attributes per resource | %d |\n", complexAttrCount)
114+
fmt.Printf("| Total attribute checks | %d |\n", complexResourceCount*complexAttrCount)
115+
fmt.Printf("| Iterations | %d |\n", iterations)
116+
fmt.Printf("\n")
117+
fmt.Printf("## Results\n")
118+
fmt.Printf("| Metric | Value |\n")
119+
fmt.Printf("|-------------------------|------------------------|\n")
120+
fmt.Printf("| Approved Decisions | %d |\n", lastApproved)
121+
fmt.Printf("| Denied Decisions | %d |\n", lastDenied)
122+
fmt.Printf("| Average Request Time | %s |\n", avgTime)
123+
fmt.Printf("| Decisions/second | %.2f |\n", decisionsPerSecond)
124+
fmt.Printf("| Time per decision | %s |\n", time.Duration(int64(avgTime)/int64(complexResourceCount)))
125+
126+
return nil
127+
}

‎lib/flattening/flatten_bench_large_test.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,5 +71,3 @@ func BenchmarkFlatten_NestedEntity(b *testing.B) {
7171
}
7272
}
7373
}
74-
75-

0 commit comments

Comments
 (0)