diff --git a/autocomplete.go b/autocomplete.go new file mode 100644 index 0000000..0505607 --- /dev/null +++ b/autocomplete.go @@ -0,0 +1,421 @@ +// Copyright 2024 RunReveal Inc. +// SPDX-License-Identifier: Apache-2.0 + +package pql + +import ( + "cmp" + "slices" + "strings" + "unicode/utf8" + + "github.com/runreveal/pql/parser" +) + +// AnalysisContext is information about the eventual execution environment +// passed in to assist in analysis tasks. +type AnalysisContext struct { + Tables map[string]*AnalysisTable +} + +// AnalysisTable is a table known to an [AnalysisContext]. +type AnalysisTable struct { + Columns []*AnalysisColumn +} + +// AnalysisColumn is a column known to an [AnalysisTable]. +type AnalysisColumn struct { + Name string +} + +// Completion is a single completion suggestion +// returned by [AnalysisContext.SuggestCompletions]. +type Completion struct { + // Label is the label that should be displayed for the completion. + // It is often the same as Text, + // but Text may include extra characters for convenience. + Label string + // Text is the full string that is being placed. + Text string + // Span is the position where Text should be placed. + // If the span's length is zero, + // then the text should be inserted for a successful completion. + // Otherwise, the span indicates text that should be replaced with the text. + Span parser.Span +} + +// SuggestCompletions suggests possible snippets to insert +// given a partial pql statement and a selected range. +func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span) []*Completion { + pos := cursor.End + + tokens := parser.Scan(source) + stmts, _ := parser.Parse(source) + prefix := completionPrefix(tokens, pos) + i := spanBefore(stmts, pos, parser.Statement.Span) + if i < 0 { + return ctx.completeTableNames(source, prefix) + } + letNames := make(map[string]struct{}) + for _, stmt := range stmts[:i] { + if stmt, ok := stmt.(*parser.LetStatement); ok && stmt.Name != nil { + letNames[stmt.Name.Name] = struct{}{} + } + } + + // Try to figure out whether we're between a semicolon and another statement. + nextTokenIndex, _ := slices.BinarySearchFunc(tokens, stmts[i].Span().End, func(tok parser.Token, i int) int { + return cmp.Compare(tok.Span.Start, i) + }) + if nextTokenIndex < len(tokens) && tokens[nextTokenIndex].Kind == parser.TokenSemi && pos >= tokens[nextTokenIndex].Span.End { + return ctx.completeTableNames(source, prefix) + } + + switch stmt := stmts[i].(type) { + case *parser.LetStatement: + return ctx.suggestLetStatement(source, stmt, letNames, prefix) + case *parser.TabularExpr: + return ctx.suggestTabularExpr(source, stmt, letNames, prefix) + default: + return nil + } +} + +func (ctx *AnalysisContext) suggestLetStatement(source string, stmt *parser.LetStatement, letNames map[string]struct{}, prefix parser.Span) []*Completion { + if !stmt.Assign.IsValid() || prefix.End < stmt.Assign.End { + return nil + } + return completeScope(source, prefix, letNames) +} + +func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.TabularExpr, letNames map[string]struct{}, prefix parser.Span) []*Completion { + if expr == nil { + return ctx.completeTableNames(source, prefix) + } + + if sourceSpan := expr.Source.Span(); prefix.Overlaps(sourceSpan) || prefix.End < sourceSpan.Start { + // Assume that this is a table name. + return ctx.completeTableNames(source, prefix) + } + + // Find the operator that this cursor is associated with. + i := spanBefore(expr.Operators, prefix.End, parser.TabularOperator.Span) + if i < 0 { + // Before the first operator. + return completeOperators(source, prefix, true) + } + + columns := ctx.determineColumnsInScope(expr.Source, expr.Operators[:i]) + + switch op := expr.Operators[i].(type) { + case *parser.UnknownTabularOperator: + if prefix.End == op.Pipe.End { + return completeOperators(source, prefix, false) + } + if name := op.Name(); name != nil && name.NameSpan.Overlaps(prefix) { + return completeOperators(source, prefix, false) + } + if len(op.Tokens) == 0 || prefix.End < op.Tokens[0].Span.Start { + return completeOperators(source, prefix, false) + } + return nil + case *parser.WhereOperator: + if prefix.End <= op.Keyword.Start { + return completeOperators(source, prefix, false) + } + if prefix.End <= op.Keyword.End { + return nil + } + return completeExpression(source, prefix, letNames, columns) + case *parser.SortOperator: + if prefix.End <= op.Keyword.Start { + return completeOperators(source, prefix, false) + } + if prefix.End <= op.Keyword.End { + return nil + } + return completeExpression(source, prefix, letNames, columns) + case *parser.TakeOperator: + if prefix.End <= op.Keyword.Start { + return completeOperators(source, prefix, false) + } + if prefix.End <= op.Keyword.End { + return nil + } + return completeScope(source, prefix, letNames) + case *parser.TopOperator: + if prefix.End <= op.Keyword.Start { + return completeOperators(source, prefix, false) + } + if !op.By.IsValid() || prefix.End <= op.By.Start { + return completeScope(source, prefix, letNames) + } + return completeExpression(source, prefix, letNames, columns) + case *parser.ProjectOperator: + if prefix.End <= op.Keyword.Start { + return completeOperators(source, prefix, false) + } + if prefix.End <= op.Keyword.End { + return nil + } + return completeExpression(source, prefix, letNames, columns) + case *parser.ExtendOperator: + if prefix.End <= op.Keyword.Start { + return completeOperators(source, prefix, false) + } + if prefix.End <= op.Keyword.End { + return nil + } + return completeExpression(source, prefix, letNames, columns) + case *parser.SummarizeOperator: + if prefix.End <= op.Keyword.Start { + return completeOperators(source, prefix, false) + } + if prefix.End <= op.Keyword.End { + return nil + } + return completeExpression(source, prefix, letNames, columns) + case *parser.JoinOperator: + if prefix.End <= op.Keyword.Start { + return completeOperators(source, prefix, false) + } + if prefix.End <= op.Keyword.End { + return nil + } + if op.Lparen.IsValid() && prefix.End >= op.Lparen.End && (!op.Rparen.IsValid() || prefix.End <= op.Rparen.Start) { + return ctx.suggestTabularExpr(source, op.Right, letNames, prefix) + } + if op.On.IsValid() && prefix.End > op.On.End { + return completeExpression(source, prefix, letNames, columns) + } + return nil + case *parser.AsOperator: + if prefix.End <= op.Keyword.Start { + return completeOperators(source, prefix, false) + } + return nil + default: + return nil + } +} + +var sortedOperatorNames = []string{ + "as", + "count", + "extend", + "join", + "limit", + "order", + "project", + "sort", + "summarize", + "take", + "top", + "where", +} + +func (ctx *AnalysisContext) determineColumnsInScope(source parser.TabularDataSource, ops []parser.TabularOperator) []*AnalysisColumn { + var columns []*AnalysisColumn + if source, ok := source.(*parser.TableRef); ok { + if tab := ctx.Tables[source.Table.Name]; tab != nil { + columns = tab.Columns + } + } + for _, op := range ops { + switch op := op.(type) { + case *parser.CountOperator: + columns = []*AnalysisColumn{{Name: "count()"}} + case *parser.ProjectOperator: + columns = make([]*AnalysisColumn, 0, len(op.Cols)) + for _, col := range op.Cols { + columns = append(columns, &AnalysisColumn{ + Name: col.Name.Name, + }) + } + case *parser.ExtendOperator: + columns = slices.Clip(columns) + for _, col := range op.Cols { + columns = append(columns, &AnalysisColumn{ + Name: col.Name.Name, + }) + } + case *parser.SummarizeOperator: + columns = make([]*AnalysisColumn, 0, len(op.Cols)+len(op.GroupBy)) + for _, col := range op.Cols { + columns = append(columns, &AnalysisColumn{ + Name: col.Name.Name, + }) + } + for _, col := range op.GroupBy { + columns = append(columns, &AnalysisColumn{ + Name: col.Name.Name, + }) + } + case *parser.JoinOperator: + columns = slices.Clip(columns) + columns = append(columns, ctx.determineColumnsInScope(op.Right.Source, op.Right.Operators)...) + } + } + return columns +} + +func completeExpression(source string, prefixSpan parser.Span, scope map[string]struct{}, columns []*AnalysisColumn) []*Completion { + return append(completeColumnNames(source, prefixSpan, columns), completeScope(source, prefixSpan, scope)...) +} + +func completeColumnNames(source string, prefixSpan parser.Span, columns []*AnalysisColumn) []*Completion { + prefix := source[prefixSpan.Start:prefixSpan.End] + + result := make([]*Completion, 0, len(columns)) + for _, col := range columns { + if hasFoldPrefix(col.Name, prefix) { + result = append(result, &Completion{ + Label: col.Name, + Text: col.Name, + Span: prefixSpan, + }) + } + } + return result +} + +func completeScope(source string, prefixSpan parser.Span, scope map[string]struct{}) []*Completion { + prefix := source[prefixSpan.Start:prefixSpan.End] + + result := make([]*Completion, 0, len(scope)) + for name := range scope { + if hasFoldPrefix(name, prefix) { + result = append(result, &Completion{ + Label: name, + Text: name, + Span: prefixSpan, + }) + } + } + return result +} + +func (ctx *AnalysisContext) completeTableNames(source string, prefixSpan parser.Span) []*Completion { + prefix := source[prefixSpan.Start:prefixSpan.End] + + result := make([]*Completion, 0, len(ctx.Tables)) + for tableName := range ctx.Tables { + if hasFoldPrefix(tableName, prefix) { + result = append(result, &Completion{ + Label: tableName, + Text: tableName, + Span: prefixSpan, + }) + } + } + slices.SortFunc(result, func(a, b *Completion) int { + return cmp.Compare(a.Text, b.Text) + }) + return result +} + +func completeOperators(source string, prefixSpan parser.Span, includePipe bool) []*Completion { + if includePipe { + // Should always be an insert, not a replacement. + prefixSpan = parser.Span{ + Start: prefixSpan.End, + End: prefixSpan.End, + } + } + prefix := source[prefixSpan.Start:prefixSpan.End] + leading := "" + if includePipe { + leading = "| " + } else if prefixSpan.Len() == 0 && prefixSpan.Start > 0 && source[prefixSpan.Start-1] == '|' { + // If directly adjacent to pipe, automatically add a space. + leading = " " + } + + result := make([]*Completion, 0, len(sortedOperatorNames)) + for _, name := range sortedOperatorNames { + if !hasFoldPrefix(name, prefix) { + continue + } + c := &Completion{ + Label: name, + Text: leading + name, + Span: prefixSpan, + } + if name == "order" || name == "sort" { + c.Text += " by" + } + result = append(result, c) + } + return result +} + +// completionPrefix returns a span of characters that should be considered for +// filtering completion results and replacement during the completion. +func completionPrefix(tokens []parser.Token, pos int) parser.Span { + result := parser.Span{ + Start: pos, + End: pos, + } + if len(tokens) == 0 { + return result + } + i := spanBefore(tokens, pos, func(tok parser.Token) parser.Span { return tok.Span }) + if i < 0 { + return result + } + if !tokens[i].Span.Overlaps(parser.Span{Start: pos, End: pos}) || !isCompletableToken(tokens[i].Kind) { + // Cursor is not adjacent to token. Assume there's whitespace. + return result + } + result.Start = tokens[i].Span.Start + if tokens[i].Kind == parser.TokenQuotedIdentifier { + // Skip past initial backtick. + result.Start += len("`") + } + return result +} + +// spanBefore finds the first span in a sorted slice +// that starts before the given position. +// The span function is used to obtain the span of each element in the slice. +// If the position occurs before any spans, +// spanBefore returns -1. +func spanBefore[S ~[]E, E any](x S, pos int, span func(E) parser.Span) int { + i, _ := slices.BinarySearchFunc(x, pos, func(elem E, pos int) int { + return cmp.Compare(span(elem).Start, pos) + }) + // Binary search will find the span that follows the position. + return i - 1 +} + +func isCompletableToken(kind parser.TokenKind) bool { + return kind == parser.TokenIdentifier || + kind == parser.TokenQuotedIdentifier || + kind == parser.TokenAnd || + kind == parser.TokenOr || + kind == parser.TokenIn || + kind == parser.TokenBy +} + +// hasFoldPrefix reports whether s starts with the given prefix, +// ignoring differences in case. +func hasFoldPrefix(s, prefix string) bool { + n := utf8.RuneCountInString(prefix) + + // Find the end of the first n runes in s. + // If s does not have that many runes, + // then it can't have the prefix. + if len(s) < n { + return false + } + var prefixLen int + for i := 0; i < n; i++ { + _, sz := utf8.DecodeRuneInString(s[prefixLen:]) + if sz == 0 { + return false + } + prefixLen += sz + } + + return strings.EqualFold(s[:prefixLen], prefix) +} diff --git a/autocomplete_test.go b/autocomplete_test.go new file mode 100644 index 0000000..99a7456 --- /dev/null +++ b/autocomplete_test.go @@ -0,0 +1,874 @@ +// Copyright 2024 RunReveal Inc. +// SPDX-License-Identifier: Apache-2.0 + +package pql + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/runreveal/pql/parser" +) + +var completionTests = []struct { + name string + + context *AnalysisContext + sourceBefore string + sourceAfter string + + want []*Completion +}{ + { + name: "Empty", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "n"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "", + sourceAfter: "", + want: []*Completion{ + { + Label: "foo", + Text: "foo", + Span: parser.Span{ + Start: 0, + End: 0, + }, + }, + { + Label: "bar", + Text: "bar", + Span: parser.Span{ + Start: 0, + End: 0, + }, + }, + }, + }, + { + name: "InitialSourceRef", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "n"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "f", + sourceAfter: "", + want: []*Completion{ + { + Label: "foo", + Text: "foo", + Span: parser.Span{ + Start: 0, + End: 1, + }, + }, + }, + }, + { + name: "SourceRefWithPipe", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "n"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "", + sourceAfter: " | count", + want: []*Completion{ + { + Label: "foo", + Text: "foo", + Span: parser.Span{ + Start: 0, + End: 0, + }, + }, + { + Label: "bar", + Text: "bar", + Span: parser.Span{ + Start: 0, + End: 0, + }, + }, + }, + }, + { + name: "BeforeCompleteExpr", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "n"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "", + sourceAfter: "o | count", + want: []*Completion{ + { + Label: "foo", + Text: "foo", + Span: parser.Span{ + Start: 0, + End: 0, + }, + }, + { + Label: "bar", + Text: "bar", + Span: parser.Span{ + Start: 0, + End: 0, + }, + }, + }, + }, + { + name: "BeforeSpaceThenCompleteExpr", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "n"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "", + sourceAfter: " x | count", + want: []*Completion{ + { + Label: "foo", + Text: "foo", + Span: parser.Span{ + Start: 0, + End: 0, + }}, + { + Label: "bar", + Text: "bar", + Span: parser.Span{ + Start: 0, + End: 0, + }}, + }, + }, + { + name: "FirstOperator", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "n"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "foo ", + sourceAfter: "", + want: []*Completion{ + { + Label: "as", + Text: "| as", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "count", + Text: "| count", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "extend", + Text: "| extend", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "join", + Text: "| join", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "limit", + Text: "| limit", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "order", + Text: "| order by", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "project", + Text: "| project", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "sort", + Text: "| sort by", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "summarize", + Text: "| summarize", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "take", + Text: "| take", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "top", + Text: "| top", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + { + Label: "where", + Text: "| where", + Span: parser.Span{ + Start: 4, + End: 4, + }, + }, + }, + }, + { + name: "FirstOperatorAfterPipe", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "n"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "foo |", + sourceAfter: "", + want: []*Completion{ + { + Label: "as", + Text: " as", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "count", + Text: " count", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "extend", + Text: " extend", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "join", + Text: " join", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "limit", + Text: " limit", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "order", + Text: " order by", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "project", + Text: " project", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "sort", + Text: " sort by", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "summarize", + Text: " summarize", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "take", + Text: " take", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "top", + Text: " top", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + { + Label: "where", + Text: " where", + Span: parser.Span{ + Start: 5, + End: 5, + }, + }, + }, + }, + { + name: "FirstOperatorAfterPipeSpace", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "n"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "foo | ", + sourceAfter: "", + want: []*Completion{ + { + Label: "as", + Text: "as", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "count", + Text: "count", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "extend", + Text: "extend", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "join", + Text: "join", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "limit", + Text: "limit", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "order", + Text: "order by", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "project", + Text: "project", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "sort", + Text: "sort by", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "summarize", + Text: "summarize", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "take", + Text: "take", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "top", + Text: "top", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + { + Label: "where", + Text: "where", + Span: parser.Span{ + Start: 7, + End: 7, + }, + }, + }, + }, + { + name: "FirstOperatorPartial", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "n"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "foo | whe", + want: []*Completion{ + { + Label: "where", + Text: "where", + Span: parser.Span{ + Start: 6, + End: 9, + }, + }, + }, + }, + { + name: "WhereExpression", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "name"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "foo | where n", + want: []*Completion{ + { + Label: "name", + Text: "name", + Span: parser.Span{ + Start: 12, + End: 13, + }, + }, + }, + }, + { + name: "JoinExpression", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "name"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "foo | join (b", + want: []*Completion{ + { + Label: "bar", + Text: "bar", + Span: parser.Span{ + Start: 12, + End: 13, + }, + }, + }, + }, + { + name: "JoinExpressionOn", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "foo": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + {Name: "name"}, + }, + }, + "bar": { + Columns: []*AnalysisColumn{ + {Name: "id"}, + }, + }, + }, + }, + sourceBefore: "foo | join (bar) on i", + want: []*Completion{ + { + Label: "id", + Text: "id", + Span: parser.Span{ + Start: 20, + End: 21, + }, + }, + }, + }, + { + name: "Project", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "People": { + Columns: []*AnalysisColumn{ + {Name: "FirstName"}, + {Name: "LastName"}, + }, + }, + }, + }, + sourceBefore: "People\n| project F", + sourceAfter: ", LastName", + want: []*Completion{ + { + Label: "FirstName", + Text: "FirstName", + Span: parser.Span{ + Start: 17, + End: 18, + }, + }, + }, + }, + { + name: "LetTake", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "People": { + Columns: []*AnalysisColumn{ + {Name: "FirstName"}, + {Name: "LastName"}, + }, + }, + }, + }, + sourceBefore: "let foo = 5;\nPeople\n| take ", + want: []*Completion{ + { + Label: "foo", + Text: "foo", + Span: parser.Span{ + Start: 27, + End: 27, + }, + }, + }, + }, + { + name: "LetWhere", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "People": { + Columns: []*AnalysisColumn{ + {Name: "FirstName"}, + {Name: "LastName"}, + }, + }, + }, + }, + sourceBefore: "let foo = \"Jane\";\nPeople\n| where FirstName = ", + want: []*Completion{ + { + Label: "foo", + Text: "foo", + Span: parser.Span{ + Start: 45, + End: 45, + }, + }, + { + Label: "FirstName", + Text: "FirstName", + Span: parser.Span{ + Start: 45, + End: 45, + }, + }, + { + Label: "LastName", + Text: "LastName", + Span: parser.Span{ + Start: 45, + End: 45, + }, + }, + }, + }, + { + name: "AfterLet", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "People": { + Columns: []*AnalysisColumn{ + {Name: "FirstName"}, + {Name: "LastName"}, + }, + }, + }, + }, + sourceBefore: "let foo = 42;\n", + want: []*Completion{ + { + Label: "People", + Text: "People", + Span: parser.Span{ + Start: 14, + End: 14, + }, + }, + }, + }, + { + name: "CaseInsensitive", + context: &AnalysisContext{ + Tables: map[string]*AnalysisTable{ + "People": { + Columns: []*AnalysisColumn{ + {Name: "FirstName"}, + {Name: "LastName"}, + }, + }, + }, + }, + sourceBefore: "People | where fI", + want: []*Completion{ + { + Label: "FirstName", + Text: "FirstName", + Span: parser.Span{ + Start: 15, + End: 17, + }, + }, + }, + }, +} + +func TestSuggestCompletions(t *testing.T) { + for _, test := range completionTests { + t.Run(test.name, func(t *testing.T) { + got := test.context.SuggestCompletions(test.sourceBefore+test.sourceAfter, parser.Span{ + Start: len(test.sourceBefore), + End: len(test.sourceBefore), + }) + completionLess := func(a, b *Completion) bool { + if a.Span.Start != b.Span.Start { + return a.Span.Start < b.Span.Start + } + if a.Span.End != b.Span.End { + return a.Span.End < b.Span.End + } + if a.Label != b.Label { + return a.Label < b.Label + } + return a.Text < b.Text + } + if diff := cmp.Diff(test.want, got, cmpopts.SortSlices(completionLess)); diff != "" { + t.Errorf("SuggestCompletions(...) (-want +got):\n%s", diff) + } + }) + } +} + +func BenchmarkSuggestCompletions(b *testing.B) { + for _, test := range completionTests { + b.Run(test.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + test.context.SuggestCompletions(test.sourceBefore+test.sourceAfter, parser.Span{ + Start: len(test.sourceBefore), + End: len(test.sourceBefore), + }) + } + }) + } +} + +func TestHasFoldPrefix(t *testing.T) { + tests := []struct { + s string + prefix string + want bool + }{ + {"", "", true}, + {"abc", "", true}, + {"abc", "a", true}, + {"abc", "A", true}, + {"Abc", "a", true}, + {"Abc", "A", true}, + {"Abc", "x", false}, + {"", "abc", false}, + {"ABC", "abc", true}, + {"abc", "ABC", true}, + {"ABC", "abcd", false}, + } + for _, test := range tests { + if got := hasFoldPrefix(test.s, test.prefix); got != test.want { + t.Errorf("hasFoldPrefix(%q, %q) = %t; want %t", test.s, test.prefix, got, test.want) + } + } +} diff --git a/parser/ast.go b/parser/ast.go index a2f8e4e..4dae47d 100644 --- a/parser/ast.go +++ b/parser/ast.go @@ -123,6 +123,39 @@ type TabularOperator interface { tabularOperator() } +// UnknownTabularOperator is a placeholder for an unrecognized [TabularOperator]. +// [Parse] will not return such an operator without an error. +type UnknownTabularOperator struct { + Pipe Span + Tokens []Token +} + +func (op *UnknownTabularOperator) tabularOperator() {} + +// Name returns the operator's name +// or nil if a name could not be extracted. +func (op *UnknownTabularOperator) Name() *Ident { + if op == nil || len(op.Tokens) == 0 || op.Tokens[0].Kind != TokenIdentifier { + return nil + } + return &Ident{ + Name: op.Tokens[0].Value, + NameSpan: op.Tokens[0].Span, + } +} + +func (op *UnknownTabularOperator) Span() Span { + if op == nil { + return nullSpan() + } + spans := make([]Span, 1, 1+len(op.Tokens)) + spans[0] = op.Pipe + for _, tok := range op.Tokens { + spans = append(spans, tok.Span) + } + return unionSpans(spans...) +} + // CountOperator represents a `| count` operator in a [TabularExpr]. // It implements [TabularOperator]. type CountOperator struct { @@ -685,6 +718,8 @@ func Walk(n Node, visit func(n Node) bool) { if visit(n) { stack = append(stack, n.Name) } + case *UnknownTabularOperator: + visit(n) case *BinaryExpr: if visit(n) { stack = append(stack, n.Y) diff --git a/parser/parser.go b/parser/parser.go index 8f34b33..2c16318 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -157,6 +157,9 @@ func (p *parser) tabularExpr() (*TabularExpr, error) { operatorName, ok := opParser.next() if !ok { + expr.Operators = append(expr.Operators, &UnknownTabularOperator{ + Pipe: pipeToken.Span, + }) finalError = joinErrors(finalError, &parseError{ source: opParser.source, span: pipeToken.Span, @@ -165,6 +168,10 @@ func (p *parser) tabularExpr() (*TabularExpr, error) { continue } if operatorName.Kind != TokenIdentifier { + expr.Operators = append(expr.Operators, &UnknownTabularOperator{ + Pipe: pipeToken.Span, + Tokens: opParser.tokens, + }) finalError = joinErrors(finalError, &parseError{ source: opParser.source, span: operatorName.Span, @@ -234,6 +241,10 @@ func (p *parser) tabularExpr() (*TabularExpr, error) { } finalError = joinErrors(finalError, err) default: + expr.Operators = append(expr.Operators, &UnknownTabularOperator{ + Pipe: pipeToken.Span, + Tokens: opParser.tokens, + }) finalError = joinErrors(finalError, &parseError{ source: opParser.source, span: operatorName.Span, diff --git a/parser/parser_test.go b/parser/parser_test.go index 794df5f..61dd721 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -1930,6 +1930,119 @@ var parserTests = []struct { }, }, }, + { + name: "TrailingPipe", + query: "X |", + want: []Statement{&TabularExpr{ + Source: &TableRef{ + Table: &Ident{ + Name: "X", + NameSpan: newSpan(0, 1), + }, + }, + Operators: []TabularOperator{ + &UnknownTabularOperator{ + Pipe: newSpan(2, 3), + }, + }, + }}, + err: true, + }, + { + name: "UnknownOperator", + query: "X | xyzzy", + want: []Statement{&TabularExpr{ + Source: &TableRef{ + Table: &Ident{ + Name: "X", + NameSpan: newSpan(0, 1), + }, + }, + Operators: []TabularOperator{ + &UnknownTabularOperator{ + Pipe: newSpan(2, 3), + Tokens: []Token{ + { + Kind: TokenIdentifier, + Span: newSpan(4, 9), + Value: "xyzzy", + }, + }, + }, + }, + }}, + err: true, + }, + { + name: "UnknownOperatorInMiddle", + query: "X | xyzzy (Y | Z) | count", + want: []Statement{&TabularExpr{ + Source: &TableRef{ + Table: &Ident{ + Name: "X", + NameSpan: newSpan(0, 1), + }, + }, + Operators: []TabularOperator{ + &UnknownTabularOperator{ + Pipe: newSpan(2, 3), + Tokens: []Token{ + { + Kind: TokenIdentifier, + Span: newSpan(4, 9), + Value: "xyzzy", + }, + { + Kind: TokenLParen, + Span: newSpan(10, 11), + }, + { + Kind: TokenIdentifier, + Span: newSpan(11, 12), + Value: "Y", + }, + { + Kind: TokenPipe, + Span: newSpan(13, 14), + }, + { + Kind: TokenIdentifier, + Span: newSpan(15, 16), + Value: "Z", + }, + { + Kind: TokenRParen, + Span: newSpan(16, 17), + }, + }, + }, + &CountOperator{ + Pipe: newSpan(18, 19), + Keyword: newSpan(20, 25), + }, + }, + }}, + err: true, + }, + { + name: "PartialProject", + query: "People | project , LastName", + want: []Statement{&TabularExpr{ + Source: &TableRef{ + Table: &Ident{ + Name: "People", + NameSpan: newSpan(0, 6), + }, + }, + Operators: []TabularOperator{ + &ProjectOperator{ + Pipe: newSpan(7, 8), + Keyword: newSpan(9, 16), + }, + }, + }}, + err: true, + }, } func TestParse(t *testing.T) { diff --git a/parser/span.go b/parser/span.go index 9d61c0b..1e690c3 100644 --- a/parser/span.go +++ b/parser/span.go @@ -47,6 +47,23 @@ func (span Span) String() string { return fmt.Sprintf("[%d,%d)", span.Start, span.End) } +// Overlaps reports whether span and span2 intersect. +// In the case where both of the spans are zero-length, +// Overlaps reports true if the spans are equal and valid. +// In the case where only one of the spans is zero-length, +// Overlaps reports true if the zero-length span's start +// is between the other span's bounds, inclusive. +func (span Span) Overlaps(span2 Span) bool { + if !span.IsValid() || !span2.IsValid() { + return false + } + intersect := Span{ + Start: max(span.Start, span2.Start), + End: min(span.End, span2.End), + } + return intersect.IsValid() +} + func unionSpans(spans ...Span) Span { u := nullSpan() for _, span := range spans {