Skip to content

Commit c3a5bbd

Browse files
SungJin1212SungJin1212
committed
Implement missing parts
Signed-off-by: SungJin1212 <[email protected]>
1 parent 5d19f58 commit c3a5bbd

34 files changed

+545
-255
lines changed

‎engine/engine.go

Lines changed: 43 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -184,14 +184,15 @@ func NewWithScanners(opts Opts, scanners engstorage.Scanners) *Engine {
184184
disableDuplicateLabelChecks: opts.DisableDuplicateLabelChecks,
185185
disableFallback: opts.DisableFallback,
186186

187-
logger: opts.Logger,
188-
lookbackDelta: opts.LookbackDelta,
189-
enablePerStepStats: opts.EnablePerStepStats,
190-
logicalOptimizers: opts.getLogicalOptimizers(),
191-
timeout: opts.Timeout,
192-
metrics: metrics,
193-
extLookbackDelta: opts.ExtLookbackDelta,
194-
enableAnalysis: opts.EnableAnalysis,
187+
logger: opts.Logger,
188+
lookbackDelta: opts.LookbackDelta,
189+
enablePerStepStats: opts.EnablePerStepStats,
190+
logicalOptimizers: opts.getLogicalOptimizers(),
191+
timeout: opts.Timeout,
192+
metrics: metrics,
193+
extLookbackDelta: opts.ExtLookbackDelta,
194+
enableAnalysis: opts.EnableAnalysis,
195+
enableDelayedNameRemoval: opts.EnableDelayedNameRemoval,
195196
noStepSubqueryIntervalFn: func(d time.Duration) time.Duration {
196197
return time.Duration(opts.NoStepSubqueryIntervalFn(d.Milliseconds()) * 1000000)
197198
},
@@ -225,6 +226,8 @@ type Engine struct {
225226
extLookbackDelta time.Duration
226227
decodingConcurrency int
227228
enableAnalysis bool
229+
enablePartialResponses bool
230+
enableDelayedNameRemoval bool
228231
noStepSubqueryIntervalFn func(time.Duration) time.Duration
229232
}
230233

@@ -290,14 +293,15 @@ func (e *Engine) NewInstantQuery(ctx context.Context, q storage.Queryable, opts
290293
return nil, err
291294
}
292295
return &compatibilityQuery{
293-
Query: &Query{exec: exec, opts: opts},
294-
engine: e,
295-
plan: lplan,
296-
ts: ts,
297-
warns: warns,
298-
t: InstantQuery,
299-
resultSort: resultSort,
300-
scanners: scanners,
296+
Query: &Query{exec: exec, opts: opts},
297+
engine: e,
298+
plan: lplan,
299+
ts: ts,
300+
warns: warns,
301+
t: InstantQuery,
302+
resultSort: resultSort,
303+
scanners: scanners,
304+
enableDelayedNameRemoval: e.enableDelayedNameRemoval,
301305
}, nil
302306
}
303307

@@ -350,8 +354,9 @@ func (e *Engine) NewInstantQueryFromPlan(ctx context.Context, q storage.Queryabl
350354
warns: warns,
351355
t: InstantQuery,
352356
// TODO(fpetkovski): Infer the sort order from the plan, ideally without copying the newResultSort function.
353-
resultSort: noSortResultSort{},
354-
scanners: scnrs,
357+
resultSort: noSortResultSort{},
358+
scanners: scnrs,
359+
enableDelayedNameRemoval: e.enableDelayedNameRemoval,
355360
}, nil
356361
}
357362

@@ -404,12 +409,13 @@ func (e *Engine) NewRangeQuery(ctx context.Context, q storage.Queryable, opts pr
404409
}
405410

406411
return &compatibilityQuery{
407-
Query: &Query{exec: exec, opts: opts},
408-
engine: e,
409-
plan: lplan,
410-
warns: warns,
411-
t: RangeQuery,
412-
scanners: scnrs,
412+
Query: &Query{exec: exec, opts: opts},
413+
engine: e,
414+
plan: lplan,
415+
warns: warns,
416+
t: RangeQuery,
417+
scanners: scnrs,
418+
enableDelayedNameRemoval: e.enableDelayedNameRemoval,
413419
}, nil
414420
}
415421

@@ -471,6 +477,8 @@ func (e *Engine) makeQueryOpts(start time.Time, end time.Time, step time.Duratio
471477
EnablePerStepStats: e.enablePerStepStats && opts.EnablePerStepStats(),
472478
ExtLookbackDelta: e.extLookbackDelta,
473479
EnableAnalysis: e.enableAnalysis,
480+
EnablePartialResponses: e.enablePartialResponses,
481+
EnableDelayedNameRemoval: e.enableDelayedNameRemoval,
474482
NoStepSubqueryIntervalFn: e.noStepSubqueryIntervalFn,
475483
DecodingConcurrency: e.decodingConcurrency,
476484
}
@@ -517,9 +525,10 @@ type compatibilityQuery struct {
517525
ts time.Time // Empty for range queries.
518526
warns annotations.Annotations
519527

520-
t QueryType
521-
resultSort resultSorter
522-
cancel context.CancelFunc
528+
t QueryType
529+
resultSort resultSorter
530+
cancel context.CancelFunc
531+
enableDelayedNameRemoval bool
523532

524533
scanners engstorage.Scanners
525534
}
@@ -561,7 +570,10 @@ func (q *compatibilityQuery) Exec(ctx context.Context) (ret *promql.Result) {
561570

562571
series := make([]promql.Series, len(resultSeries))
563572
for i, s := range resultSeries {
564-
series[i].Metric = s
573+
if s.DropName && q.enableDelayedNameRemoval {
574+
s.Metric = s.Metric.DropMetricName()
575+
}
576+
series[i] = s
565577
}
566578
loop:
567579
for {
@@ -653,6 +665,9 @@ loop:
653665
}
654666
}
655667
sort.Slice(vector, q.resultSort.comparer(&vector))
668+
if vector.ContainsSameLabelset() {
669+
return newErrResult(ret, extlabels.ErrDuplicateLabelSet)
670+
}
656671
result = vector
657672
case parser.ValueTypeScalar:
658673
v := math.NaN()

‎engine/engine_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,16 @@ func TestPromqlAcceptance(t *testing.T) {
6565

6666
engine := engine.New(engine.Opts{
6767
EngineOpts: promql.EngineOpts{
68+
Logger: promslog.NewNopLogger(),
6869
EnableAtModifier: true,
6970
EnableNegativeOffset: true,
7071
MaxSamples: 5e10,
7172
Timeout: 1 * time.Hour,
7273
NoStepSubqueryIntervalFn: func(rangeMillis int64) int64 { return 30 * time.Second.Milliseconds() },
74+
EnableDelayedNameRemoval: true,
7375
}})
7476

75-
promqltest.RunBuiltinTests(t, engine)
77+
promqltest.RunBuiltinTests(t, engine, promqltest.WithSkipEvalInfo(true), promqltest.WithSkipEvalWarn(true))
7678
}
7779

7880
func TestVectorSelectorWithGaps(t *testing.T) {

‎engine/existing_test.go

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,9 @@ func TestRangeQuery(t *testing.T) {
3434
Query: `sum_over_time(bar[30s])`,
3535
Result: promql.Matrix{
3636
promql.Series{
37-
Floats: []promql.FPoint{{F: 0, T: 0}, {F: 10, T: 60000}, {F: 1000, T: 120000}},
38-
Metric: labels.Labels{},
37+
Floats: []promql.FPoint{{F: 0, T: 0}, {F: 10, T: 60000}, {F: 1000, T: 120000}},
38+
Metric: labels.Labels{},
39+
DropName: true,
3940
},
4041
},
4142
Start: time.Unix(0, 0),
@@ -49,8 +50,9 @@ func TestRangeQuery(t *testing.T) {
4950
Query: `sum_over_time(bar[45s])`,
5051
Result: promql.Matrix{
5152
promql.Series{
52-
Floats: []promql.FPoint{{F: 0, T: 0}, {F: 11, T: 60000}, {F: 1100, T: 120000}},
53-
Metric: labels.Labels{},
53+
Floats: []promql.FPoint{{F: 0, T: 0}, {F: 11, T: 60000}, {F: 1100, T: 120000}},
54+
Metric: labels.Labels{},
55+
DropName: true,
5456
},
5557
},
5658
Start: time.Unix(0, 0),
@@ -64,8 +66,9 @@ func TestRangeQuery(t *testing.T) {
6466
Query: `sum_over_time(bar[45s])`,
6567
Result: promql.Matrix{
6668
promql.Series{
67-
Floats: []promql.FPoint{{F: 0, T: 0}, {F: 11, T: 60000}, {F: 1100, T: 120000}},
68-
Metric: labels.Labels{},
69+
Floats: []promql.FPoint{{F: 0, T: 0}, {F: 11, T: 60000}, {F: 1100, T: 120000}},
70+
Metric: labels.Labels{},
71+
DropName: true,
6972
},
7073
},
7174
Start: time.Unix(0, 0),
@@ -79,8 +82,9 @@ func TestRangeQuery(t *testing.T) {
7982
Query: `sum_over_time(bar[45s])`,
8083
Result: promql.Matrix{
8184
promql.Series{
82-
Floats: []promql.FPoint{{F: 0, T: 0}, {F: 11, T: 60000}, {F: 1100, T: 120000}, {F: 110000, T: 180000}, {F: 11000000, T: 240000}},
83-
Metric: labels.Labels{},
85+
Floats: []promql.FPoint{{F: 0, T: 0}, {F: 11, T: 60000}, {F: 1100, T: 120000}, {F: 110000, T: 180000}, {F: 11000000, T: 240000}},
86+
Metric: labels.Labels{},
87+
DropName: true,
8488
},
8589
},
8690
Start: time.Unix(0, 0),
@@ -94,8 +98,9 @@ func TestRangeQuery(t *testing.T) {
9498
Query: `sum_over_time(bar[45s])`,
9599
Result: promql.Matrix{
96100
promql.Series{
97-
Floats: []promql.FPoint{{F: 5, T: 0}, {F: 59, T: 60000}, {F: 9, T: 120000}, {F: 956, T: 180000}},
98-
Metric: labels.Labels{},
101+
Floats: []promql.FPoint{{F: 5, T: 0}, {F: 59, T: 60000}, {F: 9, T: 120000}, {F: 956, T: 180000}},
102+
Metric: labels.Labels{},
103+
DropName: true,
99104
},
100105
},
101106
Start: time.Unix(0, 0),

‎engine/user_defined_test.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ load 30s
5151

5252
expected := promql.Matrix{
5353
promql.Series{
54-
Metric: labels.EmptyLabels(),
5554
Floats: []promql.FPoint{{T: 0, F: 14}, {T: 30000, F: 14}, {T: 60000, F: 14}, {T: 90000, F: 14}},
5655
},
5756
}
@@ -122,10 +121,10 @@ func (c *vectorSelectorOperator) Next(ctx context.Context) ([]model.StepVector,
122121
return vectors, nil
123122
}
124123

125-
func (c *vectorSelectorOperator) Series(ctx context.Context) ([]labels.Labels, error) {
126-
return []labels.Labels{
127-
labels.FromStrings(labels.MetricName, "http_requests_total", "container", "a"),
128-
labels.FromStrings(labels.MetricName, "http_requests_total", "container", "b"),
124+
func (c *vectorSelectorOperator) Series(ctx context.Context) ([]promql.Series, error) {
125+
return []promql.Series{
126+
{Metric: labels.FromStrings(labels.MetricName, "http_requests_total", "container", "a")},
127+
{Metric: labels.FromStrings(labels.MetricName, "http_requests_total", "container", "b")},
129128
}, nil
130129
}
131130

‎execution/aggregate/accumulator.go

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
package aggregate
55

66
import (
7+
"errors"
78
"math"
89

910
"github.com/prometheus/prometheus/model/histogram"
@@ -70,11 +71,27 @@ func (s *sumAcc) Add(v float64, h *histogram.FloatHistogram) error {
7071
var err error
7172
if h.Schema >= s.histSum.Schema {
7273
if s.histSum, err = s.histSum.Add(h); err != nil {
74+
if errors.Is(err, histogram.ErrHistogramsIncompatibleSchema) {
75+
// skip warning
76+
return nil
77+
}
78+
if errors.Is(err, histogram.ErrHistogramsIncompatibleBounds) {
79+
// skip warning
80+
return nil
81+
}
7382
return err
7483
}
7584
} else {
7685
t := h.Copy()
77-
if _, err = t.Add(s.histSum); err != nil {
86+
if s.histSum, err = t.Add(s.histSum); err != nil {
87+
if errors.Is(err, histogram.ErrHistogramsIncompatibleSchema) {
88+
// skip warning
89+
return nil
90+
}
91+
if errors.Is(err, histogram.ErrHistogramsIncompatibleBounds) {
92+
// skip warning
93+
return nil
94+
}
7895
return err
7996
}
8097
s.histSum = t
@@ -389,6 +406,14 @@ func (a *avgAcc) AddVector(vs []float64, hs []*histogram.FloatHistogram) error {
389406
}
390407
for _, h := range hs {
391408
if err := a.Add(0, h); err != nil {
409+
if errors.Is(err, histogram.ErrHistogramsIncompatibleSchema) {
410+
// skip warning
411+
return nil
412+
}
413+
if errors.Is(err, histogram.ErrHistogramsIncompatibleBounds) {
414+
// skip warning
415+
return nil
416+
}
392417
return err
393418
}
394419
}
@@ -431,12 +456,21 @@ type statAcc struct {
431456
}
432457

433458
func (s *statAcc) Add(v float64, h *histogram.FloatHistogram) error {
459+
if h != nil {
460+
// ignore native histogram for STDVAR and STDDEV.
461+
return nil
462+
}
463+
434464
s.hasValue = true
435465
s.count++
436466

437-
delta := v - s.mean
438-
s.mean += delta / s.count
439-
s.value += delta * (v - s.mean)
467+
if math.IsNaN(v) || math.IsInf(v, 0) {
468+
s.value = math.NaN()
469+
} else {
470+
delta := v - s.mean
471+
s.mean += delta / s.count
472+
s.value += delta * (v - s.mean)
473+
}
440474
return nil
441475
}
442476

@@ -458,7 +492,7 @@ type stdDevAcc struct {
458492
statAcc
459493
}
460494

461-
func newStdDevAcc() accumulator {
495+
func newStdDevAcc() *stdDevAcc {
462496
return &stdDevAcc{}
463497
}
464498

@@ -473,11 +507,15 @@ type stdVarAcc struct {
473507
statAcc
474508
}
475509

476-
func newStdVarAcc() accumulator {
510+
func newStdVarAcc() *stdVarAcc {
477511
return &stdVarAcc{}
478512
}
479513

480514
func (s *stdVarAcc) Value() (float64, *histogram.FloatHistogram) {
515+
if math.IsNaN(s.value) {
516+
return math.NaN(), nil
517+
}
518+
481519
if s.count == 1 {
482520
return 0, nil
483521
}

‎execution/aggregate/count_values.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"time"
1313

1414
"github.com/prometheus/prometheus/model/labels"
15+
"github.com/prometheus/prometheus/promql"
1516

1617
"github.com/thanos-io/promql-engine/execution/model"
1718
"github.com/thanos-io/promql-engine/query"
@@ -32,7 +33,7 @@ type countValuesOperator struct {
3233

3334
ts []int64
3435
counts []map[int]int
35-
series []labels.Labels
36+
series []promql.Series
3637

3738
once sync.Once
3839
}
@@ -70,7 +71,7 @@ func (c *countValuesOperator) String() string {
7071
return fmt.Sprintf("[countValues] without (%v) - param (%v)", c.grouping, c.param)
7172
}
7273

73-
func (c *countValuesOperator) Series(ctx context.Context) ([]labels.Labels, error) {
74+
func (c *countValuesOperator) Series(ctx context.Context) ([]promql.Series, error) {
7475
start := time.Now()
7576
defer func() { c.AddExecutionTimeTaken(time.Since(start)) }()
7677

@@ -141,7 +142,7 @@ func (c *countValuesOperator) initSeriesOnce(ctx context.Context) error {
141142

142143
ts := make([]int64, 0)
143144
counts := make([]map[int]int, 0)
144-
series := make([]labels.Labels, 0)
145+
series := make([]promql.Series, 0)
145146

146147
b := labels.NewBuilder(labels.EmptyLabels())
147148
for {
@@ -180,7 +181,7 @@ func (c *countValuesOperator) initSeriesOnce(ctx context.Context) error {
180181
hash := lbls.Hash()
181182
outputId, ok := hashToOutputId[hash]
182183
if !ok {
183-
series = append(series, lbls)
184+
series = append(series, promql.Series{Metric: lbls})
184185
outputId = len(series) - 1
185186
hashToOutputId[hash] = outputId
186187
}

0 commit comments

Comments
 (0)