Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Query Metadata #325

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Query Metadata #325

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7508b49
Add the Query analysis bool
bazooka3000 Nov 10, 2023
727b480
storage: use the new interface
GiedriusS Nov 13, 2023
2b79679
storage: add a todo
GiedriusS Nov 13, 2023
12d9eea
added GetSeriesHints() function
bazooka3000 Nov 17, 2023
b295f97
changes done to go.mod
bazooka3000 Nov 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion execution/execution.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ import (
// New creates new physical query execution for a given query expression which represents logical plan.
// TODO(bwplotka): Add definition (could be parameters for each execution operator) we can optimize - it would represent physical plan.
func New(expr parser.Expr, queryable storage.Queryable, opts *query.Options) (model.VectorOperator, error) {
selectorPool := engstore.NewSelectorPool(queryable)
selectorPool := engstore.NewSelectorPool(queryable,opts)
hints := storage.SelectHints{
Start: opts.Start.UnixMilli(),
End: opts.End.UnixMilli(),
Expand Down
28 changes: 26 additions & 2 deletions execution/remote/operator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"sync"
"time"

"github.com/gogo/protobuf/types"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql"

Expand All @@ -17,6 +18,8 @@ import (
engstore "github.com/thanos-io/promql-engine/execution/storage"
"github.com/thanos-io/promql-engine/execution/warnings"
"github.com/thanos-io/promql-engine/query"
tstore "github.com/thanos-io/thanos/pkg/store"
"github.com/thanos-io/thanos/pkg/store/hintspb"
)

type Execution struct {
Expand Down Expand Up @@ -75,8 +78,9 @@ func (e *Execution) Explain() (me string, next []model.VectorOperator) {
}

type storageAdapter struct {
query promql.Query
opts *query.Options
query promql.Query
opts *query.Options
hintsCollector *tstore.HintsCollector

once sync.Once
err error
Expand All @@ -101,6 +105,26 @@ func (s *storageAdapter) GetSeries(ctx context.Context, _, _ int) ([]engstore.Si
return s.series, nil
}

func (s *storageAdapter) GetSeriesHints() (map[string][]hintspb.SeriesResponseHints, error) {
if s.hintsCollector == nil {
return nil, nil
}

hints := make(map[string][]hintspb.SeriesResponseHints)

for key, value := range s.hintsCollector.Hints {
for _, v := range value {
h := hintspb.SeriesResponseHints{}
if err := types.UnmarshalAny(v.GetHints(), &h); err != nil {
return nil, err
}

hints[key] = append(hints[key], h)
}
}
return hints, nil
}

func (s *storageAdapter) executeQuery(ctx context.Context) {
result := s.query.Exec(ctx)
warnings.AddToContext(result.Warnings, ctx)
Expand Down
29 changes: 27 additions & 2 deletions execution/storage/filtered_selector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,42 @@ import (
"context"
"sync"

"github.com/gogo/protobuf/types"
"github.com/prometheus/prometheus/model/labels"
tstore "github.com/thanos-io/thanos/pkg/store"
"github.com/thanos-io/thanos/pkg/store/hintspb"
)

type filteredSelector struct {
selector *seriesSelector
filter Filter
selector *seriesSelector
filter Filter
hintsCollector *tstore.HintsCollector

once sync.Once
series []SignedSeries
}

// GetSeriesHints implements SeriesSelector.
func (f *filteredSelector) GetSeriesHints() (map[string][]hintspb.SeriesResponseHints, error) {
if f.hintsCollector == nil {
return nil, nil
}

hints := make(map[string][]hintspb.SeriesResponseHints)

for key, value := range f.hintsCollector.Hints {
for _, v := range value {
h := hintspb.SeriesResponseHints{}
if err := types.UnmarshalAny(v.GetHints(), &h); err != nil {
return nil, err
}

hints[key] = append(hints[key], h)
}
}
return hints, nil
}

func NewFilteredSelector(selector *seriesSelector, filter Filter) SeriesSelector {
return &filteredSelector{
selector: selector,
Expand Down
9 changes: 6 additions & 3 deletions execution/storage/pool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/cespare/xxhash/v2"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"
"github.com/thanos-io/promql-engine/query"
)

var sep = []byte{'\xff'}
Expand All @@ -18,27 +19,29 @@ type SelectorPool struct {
selectors map[uint64]*seriesSelector

queryable storage.Queryable
opts *query.Options
}

func NewSelectorPool(queryable storage.Queryable) *SelectorPool {
func NewSelectorPool(queryable storage.Queryable,opts *query.Options) *SelectorPool {
return &SelectorPool{
selectors: make(map[uint64]*seriesSelector),
queryable: queryable,
opts : opts,
}
}

func (p *SelectorPool) GetSelector(mint, maxt, step int64, matchers []*labels.Matcher, hints storage.SelectHints) SeriesSelector {
key := hashMatchers(matchers, mint, maxt, hints)
if _, ok := p.selectors[key]; !ok {
p.selectors[key] = newSeriesSelector(p.queryable, mint, maxt, step, matchers, hints)
p.selectors[key] = newSeriesSelector(p.queryable, mint, maxt, step, matchers, hints,p.opts)
}
return p.selectors[key]
}

func (p *SelectorPool) GetFilteredSelector(mint, maxt, step int64, matchers, filters []*labels.Matcher, hints storage.SelectHints) SeriesSelector {
key := hashMatchers(matchers, mint, maxt, hints)
if _, ok := p.selectors[key]; !ok {
p.selectors[key] = newSeriesSelector(p.queryable, mint, maxt, step, matchers, hints)
p.selectors[key] = newSeriesSelector(p.queryable, mint, maxt, step, matchers, hints,p.opts)
}

return NewFilteredSelector(p.selectors[key], NewFilter(filters))
Expand Down
61 changes: 53 additions & 8 deletions execution/storage/series_selector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,23 @@ import (
"context"
"sync"

"github.com/gogo/protobuf/types"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/storage"

"github.com/thanos-io/promql-engine/execution/warnings"
"github.com/thanos-io/promql-engine/query"
tquery "github.com/thanos-io/thanos/pkg/query"
tstore "github.com/thanos-io/thanos/pkg/store"
"github.com/thanos-io/thanos/pkg/store/hintspb"
)

type SeriesSelector interface {
GetSeries(ctx context.Context, shard, numShards int) ([]SignedSeries, error)
Matchers() []*labels.Matcher
// TODO: add a method to get hints. Return nil if there are no hints.
// Hints() map[string][]hintspb.SeriesHintsResponse
GetSeriesHints() (map[string][]hintspb.SeriesResponseHints, error)
}

type SignedSeries struct {
Expand All @@ -24,25 +32,28 @@ type SignedSeries struct {
}

type seriesSelector struct {
storage storage.Queryable
mint int64
maxt int64
step int64
matchers []*labels.Matcher
hints storage.SelectHints
storage storage.Queryable
mint int64
maxt int64
step int64
matchers []*labels.Matcher
hints storage.SelectHints
opts *query.Options
hintsCollector *tstore.HintsCollector

once sync.Once
series []SignedSeries
}

func newSeriesSelector(storage storage.Queryable, mint, maxt, step int64, matchers []*labels.Matcher, hints storage.SelectHints) *seriesSelector {
func newSeriesSelector(storage storage.Queryable, mint, maxt, step int64, matchers []*labels.Matcher, hints storage.SelectHints, opts *query.Options) *seriesSelector {
return &seriesSelector{
storage: storage,
maxt: maxt,
mint: mint,
step: step,
matchers: matchers,
hints: hints,
opts: opts,
}
}

Expand All @@ -67,7 +78,20 @@ func (o *seriesSelector) loadSeries(ctx context.Context) error {
}
defer querier.Close()

seriesSet := querier.Select(ctx, false, &o.hints, o.matchers...)
var seriesSet storage.SeriesSet
var h *tstore.HintsCollector

if o.opts.EnableAnalysis {
if qwh, ok := querier.(tquery.QuerierWithHints); ok {
seriesSet, h = qwh.SelectWithHints(ctx, false, &o.hints, o.matchers...)

o.hintsCollector = h
}
}
if seriesSet == nil {
seriesSet = querier.Select(ctx, false, &o.hints, o.matchers...)
}

i := 0
for seriesSet.Next() {
s := seriesSet.At()
Expand All @@ -82,6 +106,27 @@ func (o *seriesSelector) loadSeries(ctx context.Context) error {
return seriesSet.Err()
}

func (o *seriesSelector) GetSeriesHints() (map[string][]hintspb.SeriesResponseHints, error) {
if o.hintsCollector == nil {
return nil, nil
}

hints := make(map[string][]hintspb.SeriesResponseHints)

for key, value := range o.hintsCollector.Hints {

for _,v := range value {
h := hintspb.SeriesResponseHints{}
if err := types.UnmarshalAny(v.GetHints(), &h); err != nil {
return nil, err
}

hints[key] = append(hints[key], h)
}
}
return hints, nil
}

func seriesShard(series []SignedSeries, index int, numShards int) []SignedSeries {
start := index * len(series) / numShards
end := (index + 1) * len(series) / numShards
Expand Down
57 changes: 50 additions & 7 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,30 +7,43 @@ require (
github.com/cortexproject/promqlsmith v0.0.0-20230502194647-ed3e43bb7a52
github.com/efficientgo/core v1.0.0-rc.2
github.com/go-kit/log v0.2.1
github.com/gogo/protobuf v1.3.2
github.com/google/go-cmp v0.5.9
github.com/prometheus/client_golang v1.16.0
github.com/prometheus/common v0.44.0
github.com/prometheus/prometheus v0.47.1-0.20231002142425-1492031ef2e0
github.com/prometheus/prometheus v0.47.2-0.20231009162353-f6d9c84fde6b
github.com/stretchr/testify v1.8.4
github.com/thanos-io/thanos v0.0.0-00010101000000-000000000000
github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39
go.uber.org/goleak v1.2.1
golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b
gonum.org/v1/gonum v0.12.0
)

replace github.com/thanos-io/thanos => github.com/bazooka3000/thanos v0.0.0-20231119203316-a12237a109a3

require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go v1.44.327 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dennwc/varint v1.0.0 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd // indirect
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
Expand All @@ -43,54 +56,84 @@ require (
github.com/go-openapi/strfmt v0.21.7 // indirect
github.com/go-openapi/swag v0.22.4 // indirect
github.com/go-openapi/validate v0.22.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/gogo/googleapis v1.4.0 // indirect
github.com/gogo/status v1.1.1 // indirect
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/julienschmidt/httprouter v1.3.0 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/miekg/dns v1.1.55 // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/alertmanager v0.26.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common/sigv4 v0.1.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/redis/rueidis v1.0.14-go1.18 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a // indirect
github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d // indirect
go.mongodb.org/mongo-driver v1.12.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/collector/pdata v1.0.0-rcv0014 // indirect
go.opentelemetry.io/collector/semconv v0.81.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 // indirect
go.opentelemetry.io/contrib/propagators/autoprop v0.38.0 // indirect
go.opentelemetry.io/contrib/propagators/aws v1.13.0 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.13.0 // indirect
go.opentelemetry.io/contrib/propagators/jaeger v1.13.0 // indirect
go.opentelemetry.io/contrib/propagators/ot v1.13.0 // indirect
go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/bridge/opentracing v1.16.0 // indirect
go.opentelemetry.io/otel/metric v1.16.0 // indirect
go.opentelemetry.io/otel/sdk v1.16.0 // indirect
go.opentelemetry.io/otel/trace v1.16.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.12.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
go4.org/intern v0.0.0-20230525184215-6c62f75575cb // indirect
go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect
golang.org/x/crypto v0.15.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.18.0 // indirect
golang.org/x/oauth2 v0.14.0 // indirect
golang.org/x/perf v0.0.0-20231108231503-cb71e802ccb8 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.11.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230717213848-3f92550aa753 // indirect
google.golang.org/grpc v1.56.2 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
Expand Down
Loading