From 874cf29ca203ffdc81a30e700ef309553e868c9c Mon Sep 17 00:00:00 2001 From: Michael Hoffmann Date: Wed, 16 Aug 2023 14:02:00 +0200 Subject: [PATCH] engine,execution: make explain return a struct This is a preparation so we can easily extend explanations in the future. In particular we can add cost estimations to its output and such easily. Signed-off-by: Michael Hoffmann --- engine/engine.go | 52 +++++++++++----------- engine/user_defined_test.go | 4 +- execution/aggregate/hashaggregate.go | 17 ++++--- execution/aggregate/khashaggregate.go | 13 ++++-- execution/binary/scalar.go | 7 ++- execution/binary/vector.go | 11 +++-- execution/exchange/coalesce.go | 9 ++-- execution/exchange/concurrent.go | 7 ++- execution/exchange/dedup.go | 7 ++- execution/function/absent.go | 6 ++- execution/function/histogram.go | 8 ++-- execution/function/noarg.go | 7 +-- execution/function/operator.go | 10 ++++- execution/function/relabel.go | 6 ++- execution/function/scalar.go | 6 ++- execution/model/operator.go | 10 ++++- execution/noop/operator.go | 2 +- execution/remote/operator.go | 6 ++- execution/scan/literal_selector.go | 6 ++- execution/scan/matrix_selector.go | 9 ++-- execution/scan/vector_selector.go | 6 ++- execution/step_invariant/step_invariant.go | 7 ++- execution/unary/unary.go | 7 ++- 23 files changed, 145 insertions(+), 78 deletions(-) diff --git a/engine/engine.go b/engine/engine.go index f1342e18..1a4f1d75 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -369,7 +369,7 @@ type Query struct { // Explain returns human-readable explanation of the created executor. func (q *Query) Explain() *ExplainOutputNode { // TODO(bwplotka): Explain plan and steps. - return explainVector(q.exec) + return explainVector(q.exec.Explain()) } func (q *Query) Analyze() *AnalyzeOutputNode { @@ -393,16 +393,14 @@ func analyzeVector(obsv model.ObservableVectorOperator) *AnalyzeOutputNode { } } -func explainVector(v model.VectorOperator) *ExplainOutputNode { - name, vectors := v.Explain() - +func explainVector(v model.Explanation) *ExplainOutputNode { var children []ExplainOutputNode - for _, vector := range vectors { + for _, vector := range v.Next { children = append(children, *explainVector(vector)) } return &ExplainOutputNode{ - OperatorName: name, + OperatorName: v.Operator, Children: children, } } @@ -748,26 +746,30 @@ func analyze(w io.Writer, o model.ObservableVectorOperator, indent, indentNext s } func explain(w io.Writer, o model.VectorOperator, indent, indentNext string) { - me, next := o.Explain() - _, _ = w.Write([]byte(indent)) - _, _ = w.Write([]byte(me)) - if len(next) == 0 { - _, _ = w.Write([]byte("\n")) - return - } - - if me == "[*CancellableOperator]" { - _, _ = w.Write([]byte(": ")) - explain(w, next[0], "", indentNext) - return - } - _, _ = w.Write([]byte(":\n")) + var writeExplanationRec func(ex model.Explanation, indent, indentNext string) + + writeExplanationRec = func(ex model.Explanation, indent, indentNext string) { + _, _ = w.Write([]byte(indent)) + _, _ = w.Write([]byte(ex.Operator)) + if len(ex.Next) == 0 { + _, _ = w.Write([]byte("\n")) + return + } + if ex.Operator == "[*CancellableOperator]" { + _, _ = w.Write([]byte(": ")) + writeExplanationRec(ex.Next[0], "", indentNext) + return + } + _, _ = w.Write([]byte(":\n")) - for i, n := range next { - if i == len(next)-1 { - explain(w, n, indentNext+"└──", indentNext+" ") - } else { - explain(w, n, indentNext+"├──", indentNext+"│ ") + for i, n := range ex.Next { + if i == len(ex.Next)-1 { + writeExplanationRec(n, indentNext+"└──", indentNext+" ") + } else { + writeExplanationRec(n, indentNext+"├──", indentNext+"│ ") + } } } + + writeExplanationRec(o.Explain(), indent, indentNext) } diff --git a/engine/user_defined_test.go b/engine/user_defined_test.go index 7cb27fc2..fa699d3b 100644 --- a/engine/user_defined_test.go +++ b/engine/user_defined_test.go @@ -126,6 +126,6 @@ func (c *vectorSelectorOperator) GetPool() *model.VectorPool { return c.vectors } -func (c *vectorSelectorOperator) Explain() (me string, next []model.VectorOperator) { - return "vectorSelectorOperator", nil +func (c *vectorSelectorOperator) Explain() model.Explanation { + return model.Explanation{Operator: "[*vectorSelectorOperator]"} } diff --git a/execution/aggregate/hashaggregate.go b/execution/aggregate/hashaggregate.go index 9d207dd7..c9297b92 100644 --- a/execution/aggregate/hashaggregate.go +++ b/execution/aggregate/hashaggregate.go @@ -87,16 +87,21 @@ func (a *aggregate) Analyze() (model.OperatorTelemetry, []model.ObservableVector return a, ops } -func (a *aggregate) Explain() (me string, next []model.VectorOperator) { - var ops []model.VectorOperator +func (a *aggregate) Explain() model.Explanation { + res := model.Explanation{ + Next: make([]model.Explanation, 0), + } if a.paramOp != nil { - ops = append(ops, a.paramOp) + res.Next = append(res.Next, a.paramOp.Explain()) } - ops = append(ops, a.next) + res.Next = append(res.Next, a.next.Explain()) + if a.by { - return fmt.Sprintf("[*aggregate] %v by (%v)", a.aggregation.String(), a.labels), ops + res.Operator = fmt.Sprintf("[*aggregate] %v by (%v)", a.aggregation.String(), a.labels) + } else { + res.Operator = fmt.Sprintf("[*aggregate] %v without (%v)", a.aggregation.String(), a.labels) } - return fmt.Sprintf("[*aggregate] %v without (%v)", a.aggregation.String(), a.labels), ops + return res } func (a *aggregate) Series(ctx context.Context) ([]labels.Labels, error) { diff --git a/execution/aggregate/khashaggregate.go b/execution/aggregate/khashaggregate.go index d1bada91..f96260b0 100644 --- a/execution/aggregate/khashaggregate.go +++ b/execution/aggregate/khashaggregate.go @@ -159,11 +159,18 @@ func (a *kAggregate) Analyze() (model.OperatorTelemetry, []model.ObservableVecto return a, next } -func (a *kAggregate) Explain() (me string, next []model.VectorOperator) { +func (a *kAggregate) Explain() model.Explanation { + res := model.Explanation{ + Next: []model.Explanation{a.paramOp.Explain(), a.next.Explain()}, + } + if a.by { - return fmt.Sprintf("[*kaggregate] %v by (%v)", a.aggregation.String(), a.labels), []model.VectorOperator{a.paramOp, a.next} + res.Operator = fmt.Sprintf("[*kaggregate] %v by (%v)", a.aggregation.String(), a.labels) + } else { + res.Operator = fmt.Sprintf("[*kaggregate] %v without (%v)", a.aggregation.String(), a.labels) } - return fmt.Sprintf("[*kaggregate] %v without (%v)", a.aggregation.String(), a.labels), []model.VectorOperator{a.paramOp, a.next} + + return res } func (a *kAggregate) init(ctx context.Context) error { diff --git a/execution/binary/scalar.go b/execution/binary/scalar.go index 0c1ebe4d..86708770 100644 --- a/execution/binary/scalar.go +++ b/execution/binary/scalar.go @@ -102,8 +102,11 @@ func (o *scalarOperator) Analyze() (model.OperatorTelemetry, []model.ObservableV return o, next } -func (o *scalarOperator) Explain() (me string, next []model.VectorOperator) { - return fmt.Sprintf("[*scalarOperator] %s", parser.ItemTypeStr[o.opType]), []model.VectorOperator{o.next, o.scalar} +func (o *scalarOperator) Explain() model.Explanation { + return model.Explanation{ + Operator: fmt.Sprintf("[*scalarOperator] %s", parser.ItemTypeStr[o.opType]), + Next: []model.Explanation{o.next.Explain(), o.scalar.Explain()}, + } } func (o *scalarOperator) Series(ctx context.Context) ([]labels.Labels, error) { diff --git a/execution/binary/vector.go b/execution/binary/vector.go index 76b34413..099f3bfa 100644 --- a/execution/binary/vector.go +++ b/execution/binary/vector.go @@ -95,11 +95,16 @@ func (o *vectorOperator) Analyze() (model.OperatorTelemetry, []model.ObservableV return o, next } -func (o *vectorOperator) Explain() (me string, next []model.VectorOperator) { +func (o *vectorOperator) Explain() model.Explanation { + res := model.Explanation{ + Next: []model.Explanation{o.lhs.Explain(), o.rhs.Explain()}, + } if o.matching.On { - return fmt.Sprintf("[*vectorOperator] %s %v on %v group %v", parser.ItemTypeStr[o.opType], o.matching.Card.String(), o.matching.MatchingLabels, o.matching.Include), []model.VectorOperator{o.lhs, o.rhs} + res.Operator = fmt.Sprintf("[*vectorOperator] %s %v on %v group %v", parser.ItemTypeStr[o.opType], o.matching.Card.String(), o.matching.MatchingLabels, o.matching.Include) + } else { + res.Operator = fmt.Sprintf("[*vectorOperator] %s %v ignoring %v group %v", parser.ItemTypeStr[o.opType], o.matching.Card.String(), o.matching.On, o.matching.Include) } - return fmt.Sprintf("[*vectorOperator] %s %v ignoring %v group %v", parser.ItemTypeStr[o.opType], o.matching.Card.String(), o.matching.On, o.matching.Include), []model.VectorOperator{o.lhs, o.rhs} + return res } func (o *vectorOperator) Series(ctx context.Context) ([]labels.Labels, error) { diff --git a/execution/exchange/coalesce.go b/execution/exchange/coalesce.go index cc8ce857..a6a2e6df 100644 --- a/execution/exchange/coalesce.go +++ b/execution/exchange/coalesce.go @@ -72,9 +72,12 @@ func (c *coalesce) Analyze() (model.OperatorTelemetry, []model.ObservableVectorO return c, obsOperators } -func (c *coalesce) Explain() (me string, next []model.VectorOperator) { - - return "[*coalesce]", c.operators +func (c *coalesce) Explain() model.Explanation { + res := model.Explanation{Operator: "[*coalesce]", Next: make([]model.Explanation, 0)} + for _, op := range c.operators { + res.Next = append(res.Next, op.Explain()) + } + return res } func (c *coalesce) GetPool() *model.VectorPool { diff --git a/execution/exchange/concurrent.go b/execution/exchange/concurrent.go index 35640582..6cac9e1a 100644 --- a/execution/exchange/concurrent.go +++ b/execution/exchange/concurrent.go @@ -46,8 +46,11 @@ func (c *concurrencyOperator) Analyze() (model.OperatorTelemetry, []model.Observ return c, next } -func (c *concurrencyOperator) Explain() (me string, next []model.VectorOperator) { - return fmt.Sprintf("[*concurrencyOperator(buff=%v)]", c.bufferSize), []model.VectorOperator{c.next} +func (c *concurrencyOperator) Explain() model.Explanation { + return model.Explanation{ + Operator: fmt.Sprintf("[*concurrencyOperator(buff=%v)]", c.bufferSize), + Next: []model.Explanation{c.next.Explain()}, + } } func (c *concurrencyOperator) Series(ctx context.Context) ([]labels.Labels, error) { diff --git a/execution/exchange/dedup.go b/execution/exchange/dedup.go index 7bcbdf37..de4c75b2 100644 --- a/execution/exchange/dedup.go +++ b/execution/exchange/dedup.go @@ -123,8 +123,11 @@ func (d *dedupOperator) GetPool() *model.VectorPool { return d.pool } -func (d *dedupOperator) Explain() (me string, next []model.VectorOperator) { - return "[*dedup]", []model.VectorOperator{d.next} +func (d *dedupOperator) Explain() model.Explanation { + return model.Explanation{ + Operator: "[*dedup]", + Next: []model.Explanation{d.next.Explain()}, + } } func (d *dedupOperator) loadSeries(ctx context.Context) error { diff --git a/execution/function/absent.go b/execution/function/absent.go index 59c55e72..dab3fab1 100644 --- a/execution/function/absent.go +++ b/execution/function/absent.go @@ -32,8 +32,10 @@ func (o *absentOperator) Analyze() (model.OperatorTelemetry, []model.ObservableV return o, next } -func (o *absentOperator) Explain() (me string, next []model.VectorOperator) { - return "[*absentOperator]", []model.VectorOperator{} +func (o *absentOperator) Explain() model.Explanation { + return model.Explanation{ + Operator: "[*absentOperator]", + } } func (o *absentOperator) Series(_ context.Context) ([]labels.Labels, error) { diff --git a/execution/function/histogram.go b/execution/function/histogram.go index bad31fd4..49f6c7e7 100644 --- a/execution/function/histogram.go +++ b/execution/function/histogram.go @@ -60,9 +60,11 @@ func (o *histogramOperator) Analyze() (model.OperatorTelemetry, []model.Observab return o, next } -func (o *histogramOperator) Explain() (me string, next []model.VectorOperator) { - next = []model.VectorOperator{o.scalarOp, o.vectorOp} - return fmt.Sprintf("[*functionOperator] histogram_quantile(%v)", o.funcArgs), next +func (o *histogramOperator) Explain() model.Explanation { + return model.Explanation{ + Operator: fmt.Sprintf("[*functionOperator] histogram_quantile(%v)", o.funcArgs), + Next: []model.Explanation{o.scalarOp.Explain(), o.vectorOp.Explain()}, + } } func (o *histogramOperator) Series(ctx context.Context) ([]labels.Labels, error) { diff --git a/execution/function/noarg.go b/execution/function/noarg.go index fef28130..8ff29d75 100644 --- a/execution/function/noarg.go +++ b/execution/function/noarg.go @@ -33,9 +33,10 @@ func (o *noArgFunctionOperator) Analyze() (model.OperatorTelemetry, []model.Obse return o, []model.ObservableVectorOperator{} } -func (o *noArgFunctionOperator) Explain() (me string, next []model.VectorOperator) { - - return fmt.Sprintf("[*noArgFunctionOperator] %v()", o.funcExpr.Func.Name), []model.VectorOperator{} +func (o *noArgFunctionOperator) Explain() model.Explanation { + return model.Explanation{ + Operator: fmt.Sprintf("[*noArgFunctionOperator] %v()", o.funcExpr.Func.Name), + } } func (o *noArgFunctionOperator) Series(_ context.Context) ([]labels.Labels, error) { diff --git a/execution/function/operator.go b/execution/function/operator.go index 7ccc254d..effaedc8 100644 --- a/execution/function/operator.go +++ b/execution/function/operator.go @@ -173,8 +173,14 @@ func (o *functionOperator) Analyze() (model.OperatorTelemetry, []model.Observabl return o, obsOperators } -func (o *functionOperator) Explain() (me string, next []model.VectorOperator) { - return fmt.Sprintf("[*functionOperator] %v(%v)", o.funcExpr.Func.Name, o.funcExpr.Args), o.nextOps +func (o *functionOperator) Explain() model.Explanation { + res := model.Explanation{ + Operator: fmt.Sprintf("[*functionOperator] %v(%v)", o.funcExpr.Func.Name, o.funcExpr.Args), + } + for _, op := range o.nextOps { + res.Next = append(res.Next, op.Explain()) + } + return res } func (o *functionOperator) Series(ctx context.Context) ([]labels.Labels, error) { diff --git a/execution/function/relabel.go b/execution/function/relabel.go index 5b1600c4..9c6d74cd 100644 --- a/execution/function/relabel.go +++ b/execution/function/relabel.go @@ -35,8 +35,10 @@ func (o *relabelFunctionOperator) Analyze() (model.OperatorTelemetry, []model.Ob return o, next } -func (o *relabelFunctionOperator) Explain() (me string, next []model.VectorOperator) { - return "[*relabelFunctionOperator]", []model.VectorOperator{} +func (o *relabelFunctionOperator) Explain() model.Explanation { + return model.Explanation{ + Operator: "[*relabelFunctionOperator]", + } } func (o *relabelFunctionOperator) Series(ctx context.Context) ([]labels.Labels, error) { diff --git a/execution/function/scalar.go b/execution/function/scalar.go index 9d6a8553..2cc1d2fd 100644 --- a/execution/function/scalar.go +++ b/execution/function/scalar.go @@ -28,8 +28,10 @@ func (o *scalarFunctionOperator) Analyze() (model.OperatorTelemetry, []model.Obs return o, next } -func (o *scalarFunctionOperator) Explain() (me string, next []model.VectorOperator) { - return "[*scalarFunctionOperator]", []model.VectorOperator{} +func (o *scalarFunctionOperator) Explain() model.Explanation { + return model.Explanation{ + Operator: "[*scalarFunctionOperator]", + } } func (o *scalarFunctionOperator) Series(ctx context.Context) ([]labels.Labels, error) { diff --git a/execution/model/operator.go b/execution/model/operator.go index 021a36fe..689f8297 100644 --- a/execution/model/operator.go +++ b/execution/model/operator.go @@ -58,6 +58,12 @@ type ObservableVectorOperator interface { Analyze() (OperatorTelemetry, []ObservableVectorOperator) } +type Explanation struct { + Operator string + + Next []Explanation +} + // VectorOperator performs operations on series in step by step fashion. type VectorOperator interface { // Next yields vectors of samples from all series for one or more execution steps. @@ -71,6 +77,6 @@ type VectorOperator interface { // GetPool returns pool of vectors that can be shared across operators. GetPool() *VectorPool - // Explain returns human-readable explanation of the current operator and optional nested operators. - Explain() (me string, next []VectorOperator) + // Explain returns an explanation of the current operator and optional nested operators. + Explain() Explanation } diff --git a/execution/noop/operator.go b/execution/noop/operator.go index 7e192b95..e4a2ad01 100644 --- a/execution/noop/operator.go +++ b/execution/noop/operator.go @@ -21,4 +21,4 @@ func (o operator) Series(ctx context.Context) ([]labels.Labels, error) { return func (o operator) GetPool() *model.VectorPool { return nil } -func (o operator) Explain() (me string, next []model.VectorOperator) { return "noop", nil } +func (o operator) Explain() model.Explanation { return model.Explanation{Operator: "noop"} } diff --git a/execution/remote/operator.go b/execution/remote/operator.go index 3e232fa2..05787d0c 100644 --- a/execution/remote/operator.go +++ b/execution/remote/operator.go @@ -69,8 +69,10 @@ func (e *Execution) GetPool() *model.VectorPool { return e.vectorSelector.GetPool() } -func (e *Execution) Explain() (me string, next []model.VectorOperator) { - return fmt.Sprintf("[*remoteExec] %s (%d, %d)", e.query, e.opts.Start.Unix(), e.opts.End.Unix()), nil +func (e *Execution) Explain() model.Explanation { + return model.Explanation{ + Operator: fmt.Sprintf("[*remoteExec] %s (%d, %d)", e.query, e.opts.Start.Unix(), e.opts.End.Unix()), + } } type storageAdapter struct { diff --git a/execution/scan/literal_selector.go b/execution/scan/literal_selector.go index c65192f1..7add0066 100644 --- a/execution/scan/literal_selector.go +++ b/execution/scan/literal_selector.go @@ -55,8 +55,10 @@ func (o *numberLiteralSelector) Analyze() (model.OperatorTelemetry, []model.Obse return o, nil } -func (o *numberLiteralSelector) Explain() (me string, next []model.VectorOperator) { - return fmt.Sprintf("[*numberLiteralSelector] %v", o.val), nil +func (o *numberLiteralSelector) Explain() model.Explanation { + return model.Explanation{ + Operator: fmt.Sprintf("[*numberLiteralSelector] %v", o.val), + } } func (o *numberLiteralSelector) Series(context.Context) ([]labels.Labels, error) { diff --git a/execution/scan/matrix_selector.go b/execution/scan/matrix_selector.go index 5f7cdf71..1987ca6d 100644 --- a/execution/scan/matrix_selector.go +++ b/execution/scan/matrix_selector.go @@ -108,12 +108,15 @@ func (o *matrixSelector) Analyze() (model.OperatorTelemetry, []model.ObservableV return o, nil } -func (o *matrixSelector) Explain() (me string, next []model.VectorOperator) { +func (o *matrixSelector) Explain() model.Explanation { + res := model.Explanation{} r := time.Duration(o.selectRange) * time.Millisecond if o.call != nil { - return fmt.Sprintf("[*matrixSelector] %v({%v}[%s] %v mod %v)", o.funcExpr.Func.Name, o.storage.Matchers(), r, o.shard, o.numShards), nil + res.Operator = fmt.Sprintf("[*matrixSelector] %v({%v}[%s] %v mod %v)", o.funcExpr.Func.Name, o.storage.Matchers(), r, o.shard, o.numShards) + } else { + res.Operator = fmt.Sprintf("[*matrixSelector] {%v}[%s] %v mod %v", o.storage.Matchers(), r, o.shard, o.numShards) } - return fmt.Sprintf("[*matrixSelector] {%v}[%s] %v mod %v", o.storage.Matchers(), r, o.shard, o.numShards), nil + return res } func (o *matrixSelector) Series(ctx context.Context) ([]labels.Labels, error) { diff --git a/execution/scan/vector_selector.go b/execution/scan/vector_selector.go index 4b2c4dd8..e3b03685 100644 --- a/execution/scan/vector_selector.go +++ b/execution/scan/vector_selector.go @@ -85,8 +85,10 @@ func (o *vectorSelector) Analyze() (model.OperatorTelemetry, []model.ObservableV return o, nil } -func (o *vectorSelector) Explain() (me string, next []model.VectorOperator) { - return fmt.Sprintf("[*vectorSelector] {%v} %v mod %v", o.storage.Matchers(), o.shard, o.numShards), nil +func (o *vectorSelector) Explain() model.Explanation { + return model.Explanation{ + Operator: fmt.Sprintf("[*vectorSelector] {%v} %v mod %v", o.storage.Matchers(), o.shard, o.numShards), + } } func (o *vectorSelector) Series(ctx context.Context) ([]labels.Labels, error) { diff --git a/execution/step_invariant/step_invariant.go b/execution/step_invariant/step_invariant.go index 33a2566f..580f2d3c 100644 --- a/execution/step_invariant/step_invariant.go +++ b/execution/step_invariant/step_invariant.go @@ -43,8 +43,11 @@ func (u *stepInvariantOperator) Analyze() (model.OperatorTelemetry, []model.Obse return u, next } -func (u *stepInvariantOperator) Explain() (me string, next []model.VectorOperator) { - return "[*stepInvariantOperator]", []model.VectorOperator{u.next} +func (u *stepInvariantOperator) Explain() model.Explanation { + return model.Explanation{ + Operator: "[*stepInvariantOperator]", + Next: []model.Explanation{u.next.Explain()}, + } } func NewStepInvariantOperator( diff --git a/execution/unary/unary.go b/execution/unary/unary.go index e1a0a18a..9ac36f34 100644 --- a/execution/unary/unary.go +++ b/execution/unary/unary.go @@ -22,8 +22,11 @@ type unaryNegation struct { model.OperatorTelemetry } -func (u *unaryNegation) Explain() (me string, next []model.VectorOperator) { - return "[*unaryNegation]", []model.VectorOperator{u.next} +func (u *unaryNegation) Explain() model.Explanation { + return model.Explanation{ + Operator: "[*unaryNegation]", + Next: []model.Explanation{u.next.Explain()}, + } } func NewUnaryNegation(