Skip to content

Commit 2117505

Browse files
branczbrancz
committed
Allow exporting via signed URL
This patch deprecates the pprof report type for a more generic export type, which both extends the functionality to allow for more formats to be possible in the future as well as allow either serving the download content inline in the response like the pprof report does, or if Parca is configured this way, then upload the export to object storage and serve a download URL that is a signed URL from object storage.
1 parent 9c82f75 commit 2117505

File tree

13 files changed

+2156
-623
lines changed

13 files changed

+2156
-623
lines changed

‎gen/proto/go/parca/query/v1alpha1/query.pb.go‎

Lines changed: 952 additions & 595 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎gen/proto/go/parca/query/v1alpha1/query_vtproto.pb.go‎

Lines changed: 653 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎gen/proto/swagger/parca/query/v1alpha1/query.swagger.json‎

Lines changed: 75 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,7 @@
299299
},
300300
{
301301
"name": "reportType",
302-
"description": "report_type is the type of report to return\n\n - REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels",
302+
"description": "report_type is the type of report to return\n\n - REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels\n - REPORT_TYPE_EXPORT: REPORT_TYPE_EXPORT contains the profile as a downloadable file",
303303
"in": "query",
304304
"required": false,
305305
"type": "string",
@@ -312,7 +312,8 @@
312312
"REPORT_TYPE_FLAMEGRAPH_ARROW",
313313
"REPORT_TYPE_SOURCE",
314314
"REPORT_TYPE_TABLE_ARROW",
315-
"REPORT_TYPE_PROFILE_METADATA"
315+
"REPORT_TYPE_PROFILE_METADATA",
316+
"REPORT_TYPE_EXPORT"
316317
],
317318
"default": "REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED"
318319
},
@@ -390,6 +391,18 @@
390391
"in": "query",
391392
"required": false,
392393
"type": "boolean"
394+
},
395+
{
396+
"name": "exportFormat",
397+
"description": "the format to respond the data in for the export\n\n - FORMAT_UNSPECIFIED: FORMAT_UNSPECIFIED unspecified\n - FORMAT_PPROF: DOWNLOAD_FORMAT_PPROF pprof format",
398+
"in": "query",
399+
"required": false,
400+
"type": "string",
401+
"enum": [
402+
"FORMAT_UNSPECIFIED",
403+
"FORMAT_PPROF"
404+
],
405+
"default": "FORMAT_UNSPECIFIED"
393406
}
394407
],
395408
"tags": [
@@ -581,10 +594,11 @@
581594
"REPORT_TYPE_FLAMEGRAPH_ARROW",
582595
"REPORT_TYPE_SOURCE",
583596
"REPORT_TYPE_TABLE_ARROW",
584-
"REPORT_TYPE_PROFILE_METADATA"
597+
"REPORT_TYPE_PROFILE_METADATA",
598+
"REPORT_TYPE_EXPORT"
585599
],
586600
"default": "REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED",
587-
"description": "- REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels",
601+
"description": "- REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels\n - REPORT_TYPE_EXPORT: REPORT_TYPE_EXPORT contains the profile as a downloadable file",
588602
"title": "ReportType is the type of report to return"
589603
},
590604
"metastorev1alpha1Function": {
@@ -813,6 +827,10 @@
813827
"$ref": "#/definitions/v1alpha1Filter"
814828
},
815829
"title": "a set of filter to apply to the query request"
830+
},
831+
"exportFormat": {
832+
"$ref": "#/definitions/v1alpha1Format",
833+
"title": "the format to respond the data in for the export"
816834
}
817835
},
818836
"title": "QueryRequest is a request for a profile query"
@@ -853,6 +871,10 @@
853871
"$ref": "#/definitions/v1alpha1ProfileMetadata",
854872
"title": "profile_metadata contains metadata about the profile i.e. binaries, labels"
855873
},
874+
"export": {
875+
"$ref": "#/definitions/v1alpha1Export",
876+
"description": "export is the response for an export request."
877+
},
856878
"total": {
857879
"type": "string",
858880
"format": "int64",
@@ -1068,6 +1090,24 @@
10681090
},
10691091
"title": "DiffProfile contains parameters for a profile diff request"
10701092
},
1093+
"v1alpha1Export": {
1094+
"type": "object",
1095+
"properties": {
1096+
"format": {
1097+
"$ref": "#/definitions/v1alpha1Format",
1098+
"description": "The format of the download."
1099+
},
1100+
"inline": {
1101+
"$ref": "#/definitions/v1alpha1InlineExport",
1102+
"description": "InlineExport is the response for an export request where actual content\nis inlined in the response."
1103+
},
1104+
"url": {
1105+
"$ref": "#/definitions/v1alpha1URLExport",
1106+
"description": "URL is the response for an export request where actual content\nis to be downloaded from the provided URL."
1107+
}
1108+
},
1109+
"description": "Export is the response for an export request."
1110+
},
10711111
"v1alpha1Filter": {
10721112
"type": "object",
10731113
"properties": {
@@ -1256,6 +1296,16 @@
12561296
},
12571297
"title": "FlamegraphRootNode is a root node of a flame graph"
12581298
},
1299+
"v1alpha1Format": {
1300+
"type": "string",
1301+
"enum": [
1302+
"FORMAT_UNSPECIFIED",
1303+
"FORMAT_PPROF"
1304+
],
1305+
"default": "FORMAT_UNSPECIFIED",
1306+
"description": "- FORMAT_UNSPECIFIED: FORMAT_UNSPECIFIED unspecified\n - FORMAT_PPROF: DOWNLOAD_FORMAT_PPROF pprof format",
1307+
"title": "format is the format to offer the data in for the download"
1308+
},
12591309
"v1alpha1FrameFilter": {
12601310
"type": "object",
12611311
"properties": {
@@ -1289,6 +1339,17 @@
12891339
},
12901340
"title": "GroupBy encapsulates the repeated fields to group by"
12911341
},
1342+
"v1alpha1InlineExport": {
1343+
"type": "object",
1344+
"properties": {
1345+
"content": {
1346+
"type": "string",
1347+
"format": "byte",
1348+
"description": "Content of the export."
1349+
}
1350+
},
1351+
"description": "InlineExport is the response for an export request where actual content is\ninlined in the response."
1352+
},
12921353
"v1alpha1LabelSet": {
12931354
"type": "object",
12941355
"properties": {
@@ -1677,6 +1738,16 @@
16771738
},
16781739
"title": "TopNodeMeta is the metadata for a given node"
16791740
},
1741+
"v1alpha1URLExport": {
1742+
"type": "object",
1743+
"properties": {
1744+
"url": {
1745+
"type": "string",
1746+
"description": "The URL to download the content from."
1747+
}
1748+
},
1749+
"description": "URLExport is the response for an export request where actual content is\nto be downloaded from the provided URL."
1750+
},
16801751
"v1alpha1ValuesResponse": {
16811752
"type": "object",
16821753
"properties": {

‎gen/proto/swagger/parca/share/v1alpha1/share.swagger.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,11 @@
2828
"REPORT_TYPE_FLAMEGRAPH_ARROW",
2929
"REPORT_TYPE_SOURCE",
3030
"REPORT_TYPE_TABLE_ARROW",
31-
"REPORT_TYPE_PROFILE_METADATA"
31+
"REPORT_TYPE_PROFILE_METADATA",
32+
"REPORT_TYPE_EXPORT"
3233
],
3334
"default": "REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED",
34-
"description": "- REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels",
35+
"description": "- REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED: REPORT_TYPE_FLAMEGRAPH_UNSPECIFIED unspecified\n - REPORT_TYPE_PPROF: REPORT_TYPE_PPROF unspecified\n - REPORT_TYPE_TOP: REPORT_TYPE_TOP unspecified\n - REPORT_TYPE_CALLGRAPH: REPORT_TYPE_CALLGRAPH unspecified\n - REPORT_TYPE_FLAMEGRAPH_TABLE: REPORT_TYPE_FLAMEGRAPH_TABLE unspecified\n - REPORT_TYPE_FLAMEGRAPH_ARROW: REPORT_TYPE_FLAMEGRAPH_ARROW unspecified\n - REPORT_TYPE_SOURCE: REPORT_TYPE_SOURCE contains source code annotated with profiling information\n - REPORT_TYPE_TABLE_ARROW: REPORT_TYPE_TABLE_ARROW unspecified\n - REPORT_TYPE_PROFILE_METADATA: REPORT_TYPE_PROFILE_METADATA contains metadata about the profile i.e. binaries, labels\n - REPORT_TYPE_EXPORT: REPORT_TYPE_EXPORT contains the profile as a downloadable file",
3536
"title": "ReportType is the type of report to return"
3637
},
3738
"metastorev1alpha1Function": {

‎parca.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@ object_storage:
99
# Scraping tends to produce a lot more data than Parca Agent CPU which
1010
# results in memory usage by the server.
1111
#
12-
# scrape_configs:
13-
# - job_name: "parca"
14-
# scrape_interval: "10s"
15-
# static_configs:
16-
# - targets: [ '127.0.0.1:7070' ]
12+
scrape_configs:
13+
- job_name: "parca"
14+
scrape_interval: "10s"
15+
static_configs:
16+
- targets: [ '127.0.0.1:7070' ]
1717

1818
# Nested under the job config:
1919
#

‎pkg/parca/parca.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@ type Flags struct {
110110

111111
Storage FlagsStorage `embed:"" prefix:"storage-"`
112112

113+
ObjectStorageExport bool `default:"false" help:"Export profiles to object storage first before serving them via a signed URL."`
114+
113115
Symbolizer FlagsSymbolizer `embed:"" prefix:"symbolizer-"`
114116

115117
Debuginfo FlagsDebuginfo `embed:"" prefix:"debuginfo-"`
@@ -424,6 +426,9 @@ func Run(ctx context.Context, logger log.Logger, reg *prometheus.Registry, flags
424426
debuginfoBucket,
425427
debuginfodClients,
426428
),
429+
flags.ObjectStorageExport,
430+
signedRequestsClient,
431+
bucket,
427432
)
428433

429434
t := telemetryservice.NewTelemetry(

‎pkg/query/columnquery.go‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ import (
1818
"context"
1919
"errors"
2020
"fmt"
21+
"io"
22+
"path"
23+
"strconv"
2124
"strings"
2225
"sync"
2326
"time"
@@ -27,6 +30,7 @@ import (
2730
"github.com/apache/arrow/go/v16/arrow/bitutil"
2831
"github.com/apache/arrow/go/v16/arrow/math"
2932
"github.com/apache/arrow/go/v16/arrow/memory"
33+
"github.com/cespare/xxhash/v2"
3034
"github.com/go-kit/log"
3135
"go.opentelemetry.io/otel/attribute"
3236
"go.opentelemetry.io/otel/trace"
@@ -62,6 +66,14 @@ type SourceFinder interface {
6266
SourceExists(ctx context.Context, ref *pb.SourceReference) (bool, error)
6367
}
6468

69+
type SignedGetClient interface {
70+
SignedGET(ctx context.Context, objectKey string, expiry time.Time) (string, error)
71+
}
72+
73+
type ObjectStoreUploadClient interface {
74+
Upload(ctx context.Context, objectKey string, r io.Reader) error
75+
}
76+
6577
// ColumnQueryAPI is the read api interface for parca
6678
// It implements the proto/query/query.proto APIServer interface.
6779
type ColumnQueryAPI struct {
@@ -77,6 +89,13 @@ type ColumnQueryAPI struct {
7789
converter *parcacol.ArrowToProfileConverter
7890

7991
sourceFinder SourceFinder
92+
93+
// Whether to first upload an export to object storage and returning a
94+
// signed URL to the client instead of serving the export directly within
95+
// the API response.
96+
objectStorgeExport bool
97+
signedGetClient SignedGetClient
98+
objectStore ObjectStoreUploadClient
8099
}
81100

82101
func NewColumnQueryAPI(
@@ -87,6 +106,9 @@ func NewColumnQueryAPI(
87106
mem memory.Allocator,
88107
converter *parcacol.ArrowToProfileConverter,
89108
sourceFinder SourceFinder,
109+
objectStorageExport bool,
110+
signedGetClient SignedGetClient,
111+
objectStore ObjectStoreUploadClient,
90112
) *ColumnQueryAPI {
91113
return &ColumnQueryAPI{
92114
logger: logger,
@@ -97,6 +119,9 @@ func NewColumnQueryAPI(
97119
mem: mem,
98120
converter: converter,
99121
sourceFinder: sourceFinder,
122+
objectStorgeExport: objectStorageExport,
123+
signedGetClient: signedGetClient,
124+
objectStore: objectStore,
100125
}
101126
}
102127

@@ -320,16 +345,24 @@ func (q *ColumnQueryAPI) Query(ctx context.Context, req *pb.QueryRequest) (*pb.Q
320345
return nil, fmt.Errorf("filtering profile: %w", err)
321346
}
322347

348+
data, err := req.MarshalVT()
349+
if err != nil {
350+
return nil, fmt.Errorf("marshaling request: %w", err)
351+
}
352+
requestKey := xxhash.Sum64(data)
353+
323354
return q.renderReport(
324355
ctx,
325356
p,
357+
strconv.FormatUint(requestKey, 16),
326358
req.GetReportType(),
327359
req.GetNodeTrimThreshold(),
328360
filtered,
329361
groupBy,
330362
req.GetSourceReference(),
331363
source,
332364
isDiff,
365+
req.GetExportFormat(),
333366
)
334367
}
335368

@@ -474,17 +507,20 @@ func filterRecord(
474507
func (q *ColumnQueryAPI) renderReport(
475508
ctx context.Context,
476509
p profile.Profile,
510+
requestKey string,
477511
typ pb.QueryRequest_ReportType,
478512
nodeTrimThreshold float32,
479513
filtered int64,
480514
groupBy []string,
481515
sourceReference *pb.SourceReference,
482516
source string,
483517
isDiff bool,
518+
exportFormat pb.Format,
484519
) (*pb.QueryResponse, error) {
485520
return RenderReport(
486521
ctx,
487522
q.tracer,
523+
requestKey,
488524
p,
489525
typ,
490526
nodeTrimThreshold,
@@ -496,12 +532,17 @@ func (q *ColumnQueryAPI) renderReport(
496532
sourceReference,
497533
source,
498534
isDiff,
535+
exportFormat,
536+
q.objectStorgeExport,
537+
q.signedGetClient,
538+
q.objectStore,
499539
)
500540
}
501541

502542
func RenderReport(
503543
ctx context.Context,
504544
tracer trace.Tracer,
545+
requestKey string,
505546
p profile.Profile,
506547
typ pb.QueryRequest_ReportType,
507548
nodeTrimThreshold float32,
@@ -513,6 +554,10 @@ func RenderReport(
513554
sourceReference *pb.SourceReference,
514555
source string,
515556
isDiff bool,
557+
exportFormat pb.Format,
558+
objectStorageExport bool,
559+
signedGetClient SignedGetClient,
560+
objectStore ObjectStoreUploadClient,
516561
) (*pb.QueryResponse, error) {
517562
ctx, span := tracer.Start(ctx, "renderReport")
518563
span.SetAttributes(attribute.String("reportType", typ.String()))
@@ -609,6 +654,53 @@ func RenderReport(
609654
Filtered: filtered,
610655
Report: &pb.QueryResponse_Pprof{Pprof: buf},
611656
}, nil
657+
case pb.QueryRequest_REPORT_TYPE_EXPORT:
658+
var buf []byte
659+
660+
switch exportFormat {
661+
case pb.Format_FORMAT_PPROF:
662+
pp, err := GenerateFlatPprof(ctx, isDiff, p)
663+
if err != nil {
664+
return nil, status.Errorf(codes.Internal, "failed to generate pprof: %v", err.Error())
665+
}
666+
667+
buf, err = SerializePprof(pp)
668+
if err != nil {
669+
return nil, status.Errorf(codes.Internal, "failed to generate pprof: %v", err.Error())
670+
}
671+
default:
672+
return nil, status.Errorf(codes.InvalidArgument, "requested export format does not exist")
673+
}
674+
675+
if objectStorageExport {
676+
key := path.Join(requestKey, "pprof")
677+
if err := objectStore.Upload(ctx, key, bytes.NewReader(buf)); err != nil {
678+
return nil, status.Errorf(codes.Internal, "failed to upload export to object storage: %v", err.Error())
679+
}
680+
681+
signedURL, err := signedGetClient.SignedGET(ctx, key, time.Now().Add(time.Hour))
682+
if err != nil {
683+
return nil, status.Errorf(codes.Internal, "failed to generate signed URL: %v", err.Error())
684+
}
685+
686+
return &pb.QueryResponse{
687+
Report: &pb.QueryResponse_Export{
688+
Export: &pb.Export{
689+
Format: exportFormat,
690+
Content: &pb.Export_Url{Url: &pb.URLExport{Url: signedURL}},
691+
},
692+
},
693+
}, nil
694+
}
695+
696+
return &pb.QueryResponse{
697+
Report: &pb.QueryResponse_Export{
698+
Export: &pb.Export{
699+
Format: exportFormat,
700+
Content: &pb.Export_Inline{Inline: &pb.InlineExport{Content: buf}},
701+
},
702+
},
703+
}, nil
612704
case pb.QueryRequest_REPORT_TYPE_TOP:
613705
op, err := converter.Convert(ctx, p)
614706
if err != nil {

0 commit comments

Comments
 (0)