From 1117ccd58df9e66bd2bd832b15ad590fd04ef77f Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 4 Jun 2024 10:08:09 -0700
Subject: [PATCH 01/16] Add basic autocomplete API

Completes table names on the first part of the expression.
---
 autocomplete.go      |  95 ++++++++++++++++++++++++++++++++++++
 autocomplete_test.go | 111 +++++++++++++++++++++++++++++++++++++++++++
 parser/span.go       |  17 +++++++
 3 files changed, 223 insertions(+)
 create mode 100644 autocomplete.go
 create mode 100644 autocomplete_test.go

diff --git a/autocomplete.go b/autocomplete.go
new file mode 100644
index 0000000..288d6fb
--- /dev/null
+++ b/autocomplete.go
@@ -0,0 +1,95 @@
+// Copyright 2024 RunReveal Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package pql
+
+import (
+	"cmp"
+	"slices"
+	"strings"
+
+	"github.com/runreveal/pql/parser"
+)
+
+type AnalysisContext struct {
+	Tables map[string]*AnalysisTable
+}
+
+type AnalysisTable struct {
+	Columns []*AnalysisColumn
+}
+
+type AnalysisColumn struct {
+	Name string
+}
+
+type Completion struct {
+	Identifier string
+}
+
+func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span) []*Completion {
+	pos := cursor.End
+	posSpan := parser.Span{Start: pos, End: pos}
+
+	tokens := parser.Scan(source)
+	expr, _ := parser.Parse(source)
+	if expr == nil {
+		prefix := completionPrefix(source, tokens, pos)
+		result := make([]*Completion, 0, len(ctx.Tables))
+		for tableName := range ctx.Tables {
+			if strings.HasPrefix(tableName, prefix) {
+				result = append(result, &Completion{
+					Identifier: tableName,
+				})
+			}
+		}
+		return result
+	}
+
+	if posSpan.Overlaps(expr.Source.Span()) {
+		// Assume that this is a table name.
+		prefix := completionPrefix(source, tokens, pos)
+		result := make([]*Completion, 0, len(ctx.Tables))
+		for tableName := range ctx.Tables {
+			if strings.HasPrefix(tableName, prefix) {
+				result = append(result, &Completion{
+					Identifier: tableName,
+				})
+			}
+		}
+		return result
+	}
+
+	// TODO(now): More.
+
+	return nil
+}
+
+func completionPrefix(source string, tokens []parser.Token, pos int) string {
+	if len(tokens) == 0 {
+		return ""
+	}
+	i, _ := slices.BinarySearchFunc(tokens, pos, func(tok parser.Token, pos int) int {
+		return cmp.Compare(tok.Span.Start, pos)
+	})
+	i = min(i, len(tokens)-1)
+	if tokens[i].Span.End < pos || !isCompletableToken(tokens[i].Kind) {
+		// Cursor is not adjacent to token. Assume there's whitespace.
+		return ""
+	}
+	start := tokens[i].Span.Start
+	if tokens[i].Kind == parser.TokenQuotedIdentifier {
+		// Skip past initial backtick.
+		start += len("`")
+	}
+	return source[start:pos]
+}
+
+func isCompletableToken(kind parser.TokenKind) bool {
+	return kind == parser.TokenIdentifier ||
+		kind == parser.TokenQuotedIdentifier ||
+		kind == parser.TokenAnd ||
+		kind == parser.TokenOr ||
+		kind == parser.TokenIn ||
+		kind == parser.TokenBy
+}
diff --git a/autocomplete_test.go b/autocomplete_test.go
new file mode 100644
index 0000000..a642df2
--- /dev/null
+++ b/autocomplete_test.go
@@ -0,0 +1,111 @@
+// Copyright 2024 RunReveal Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package pql
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/runreveal/pql/parser"
+)
+
+func TestSuggestCompletions(t *testing.T) {
+	tests := []struct {
+		name string
+
+		context      *AnalysisContext
+		sourceBefore string
+		sourceAfter  string
+
+		want []*Completion
+	}{
+		{
+			name: "Empty",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "n"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "",
+			sourceAfter:  "",
+			want: []*Completion{
+				{Identifier: "foo"},
+				{Identifier: "bar"},
+			},
+		},
+		{
+			name: "InitialSourceRef",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "n"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "f",
+			sourceAfter:  "",
+			want: []*Completion{
+				{Identifier: "foo"},
+			},
+		},
+		{
+			name: "SourceRefWithPipe",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "n"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "",
+			sourceAfter:  " | count",
+			want: []*Completion{
+				{Identifier: "foo"},
+				{Identifier: "bar"},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := test.context.SuggestCompletions(test.sourceBefore+test.sourceAfter, parser.Span{
+				Start: len(test.sourceBefore),
+				End:   len(test.sourceBefore),
+			})
+			completionLess := func(a, b *Completion) bool {
+				return a.Identifier < b.Identifier
+			}
+			if diff := cmp.Diff(test.want, got, cmpopts.SortSlices(completionLess)); diff != "" {
+				t.Errorf("SuggestCompletions(...) (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
diff --git a/parser/span.go b/parser/span.go
index 9d61c0b..1e690c3 100644
--- a/parser/span.go
+++ b/parser/span.go
@@ -47,6 +47,23 @@ func (span Span) String() string {
 	return fmt.Sprintf("[%d,%d)", span.Start, span.End)
 }
 
+// Overlaps reports whether span and span2 intersect.
+// In the case where both of the spans are zero-length,
+// Overlaps reports true if the spans are equal and valid.
+// In the case where only one of the spans is zero-length,
+// Overlaps reports true if the zero-length span's start
+// is between the other span's bounds, inclusive.
+func (span Span) Overlaps(span2 Span) bool {
+	if !span.IsValid() || !span2.IsValid() {
+		return false
+	}
+	intersect := Span{
+		Start: max(span.Start, span2.Start),
+		End:   min(span.End, span2.End),
+	}
+	return intersect.IsValid()
+}
+
 func unionSpans(spans ...Span) Span {
 	u := nullSpan()
 	for _, span := range spans {

From 4d9367d6d5447aa089135f1b8f755386497ed1c5 Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 4 Jun 2024 10:14:02 -0700
Subject: [PATCH 02/16] Add a couple more edge cases around first completion

---
 autocomplete.go      |  4 ++--
 autocomplete_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/autocomplete.go b/autocomplete.go
index 288d6fb..5461de4 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -46,7 +46,7 @@ func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span
 		return result
 	}
 
-	if posSpan.Overlaps(expr.Source.Span()) {
+	if sourceSpan := expr.Source.Span(); posSpan.Overlaps(sourceSpan) || pos < sourceSpan.Start {
 		// Assume that this is a table name.
 		prefix := completionPrefix(source, tokens, pos)
 		result := make([]*Completion, 0, len(ctx.Tables))
@@ -73,7 +73,7 @@ func completionPrefix(source string, tokens []parser.Token, pos int) string {
 		return cmp.Compare(tok.Span.Start, pos)
 	})
 	i = min(i, len(tokens)-1)
-	if tokens[i].Span.End < pos || !isCompletableToken(tokens[i].Kind) {
+	if !tokens[i].Span.Overlaps(parser.Span{Start: pos, End: pos}) || !isCompletableToken(tokens[i].Kind) {
 		// Cursor is not adjacent to token. Assume there's whitespace.
 		return ""
 	}
diff --git a/autocomplete_test.go b/autocomplete_test.go
index a642df2..8913cc8 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -92,6 +92,54 @@ func TestSuggestCompletions(t *testing.T) {
 				{Identifier: "bar"},
 			},
 		},
+		{
+			name: "BeforeCompleteExpr",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "n"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "",
+			sourceAfter:  "o | count",
+			want: []*Completion{
+				{Identifier: "foo"},
+				{Identifier: "bar"},
+			},
+		},
+		{
+			name: "BeforeSpaceThenCompleteExpr",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "n"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "",
+			sourceAfter:  " x | count",
+			want: []*Completion{
+				{Identifier: "foo"},
+				{Identifier: "bar"},
+			},
+		},
 	}
 
 	for _, test := range tests {

From 7ae9c1a5ffbef9b5bff59e4bd327035c0c641531 Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 4 Jun 2024 10:52:08 -0700
Subject: [PATCH 03/16] Add placeholder for unknown tabular operators

---
 parser/ast.go         | 35 ++++++++++++++++
 parser/parser.go      | 11 +++++
 parser/parser_test.go | 94 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 140 insertions(+)

diff --git a/parser/ast.go b/parser/ast.go
index 0acd0fe..7443cd5 100644
--- a/parser/ast.go
+++ b/parser/ast.go
@@ -115,6 +115,39 @@ type TabularOperator interface {
 	tabularOperator()
 }
 
+// UnknownTabularOperator is a placeholder for an unrecognized [TabularOperator].
+// [Parse] will not return such an operator without an error.
+type UnknownTabularOperator struct {
+	Pipe   Span
+	Tokens []Token
+}
+
+func (op *UnknownTabularOperator) tabularOperator() {}
+
+// Name returns the operator's name
+// or nil if a name could not be extracted.
+func (op *UnknownTabularOperator) Name() *Ident {
+	if op == nil || len(op.Tokens) == 0 || op.Tokens[0].Kind != TokenIdentifier {
+		return nil
+	}
+	return &Ident{
+		Name:     op.Tokens[0].Value,
+		NameSpan: op.Tokens[0].Span,
+	}
+}
+
+func (op *UnknownTabularOperator) Span() Span {
+	if op == nil {
+		return nullSpan()
+	}
+	spans := make([]Span, 1, 1+len(op.Tokens))
+	spans[0] = op.Pipe
+	for _, tok := range op.Tokens {
+		spans = append(spans, tok.Span)
+	}
+	return unionSpans(spans...)
+}
+
 // CountOperator represents a `| count` operator in a [TabularExpr].
 // It implements [TabularOperator].
 type CountOperator struct {
@@ -655,6 +688,8 @@ func Walk(n Node, visit func(n Node) bool) {
 			if visit(n) {
 				stack = append(stack, n.Name)
 			}
+		case *UnknownTabularOperator:
+			visit(n)
 		case *BinaryExpr:
 			if visit(n) {
 				stack = append(stack, n.Y)
diff --git a/parser/parser.go b/parser/parser.go
index 5c82cd0..65c381b 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -78,6 +78,9 @@ func (p *parser) tabularExpr() (*TabularExpr, error) {
 
 		operatorName, ok := opParser.next()
 		if !ok {
+			expr.Operators = append(expr.Operators, &UnknownTabularOperator{
+				Pipe: pipeToken.Span,
+			})
 			finalError = joinErrors(finalError, &parseError{
 				source: opParser.source,
 				span:   pipeToken.Span,
@@ -86,6 +89,10 @@ func (p *parser) tabularExpr() (*TabularExpr, error) {
 			continue
 		}
 		if operatorName.Kind != TokenIdentifier {
+			expr.Operators = append(expr.Operators, &UnknownTabularOperator{
+				Pipe:   pipeToken.Span,
+				Tokens: opParser.tokens,
+			})
 			finalError = joinErrors(finalError, &parseError{
 				source: opParser.source,
 				span:   operatorName.Span,
@@ -155,6 +162,10 @@ func (p *parser) tabularExpr() (*TabularExpr, error) {
 			}
 			finalError = joinErrors(finalError, err)
 		default:
+			expr.Operators = append(expr.Operators, &UnknownTabularOperator{
+				Pipe:   pipeToken.Span,
+				Tokens: opParser.tokens,
+			})
 			finalError = joinErrors(finalError, &parseError{
 				source: opParser.source,
 				span:   operatorName.Span,
diff --git a/parser/parser_test.go b/parser/parser_test.go
index 0a0bb5d..90f56a2 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -1801,6 +1801,100 @@ var parserTests = []struct {
 			},
 		},
 	},
+	{
+		name:  "TrailingPipe",
+		query: "X |",
+		want: &TabularExpr{
+			Source: &TableRef{
+				Table: &Ident{
+					Name:     "X",
+					NameSpan: newSpan(0, 1),
+				},
+			},
+			Operators: []TabularOperator{
+				&UnknownTabularOperator{
+					Pipe: newSpan(2, 3),
+				},
+			},
+		},
+		err: true,
+	},
+	{
+		name:  "UnknownOperator",
+		query: "X | xyzzy",
+		want: &TabularExpr{
+			Source: &TableRef{
+				Table: &Ident{
+					Name:     "X",
+					NameSpan: newSpan(0, 1),
+				},
+			},
+			Operators: []TabularOperator{
+				&UnknownTabularOperator{
+					Pipe: newSpan(2, 3),
+					Tokens: []Token{
+						{
+							Kind:  TokenIdentifier,
+							Span:  newSpan(4, 9),
+							Value: "xyzzy",
+						},
+					},
+				},
+			},
+		},
+		err: true,
+	},
+	{
+		name:  "UnknownOperatorInMiddle",
+		query: "X | xyzzy (Y | Z) | count",
+		want: &TabularExpr{
+			Source: &TableRef{
+				Table: &Ident{
+					Name:     "X",
+					NameSpan: newSpan(0, 1),
+				},
+			},
+			Operators: []TabularOperator{
+				&UnknownTabularOperator{
+					Pipe: newSpan(2, 3),
+					Tokens: []Token{
+						{
+							Kind:  TokenIdentifier,
+							Span:  newSpan(4, 9),
+							Value: "xyzzy",
+						},
+						{
+							Kind: TokenLParen,
+							Span: newSpan(10, 11),
+						},
+						{
+							Kind:  TokenIdentifier,
+							Span:  newSpan(11, 12),
+							Value: "Y",
+						},
+						{
+							Kind: TokenPipe,
+							Span: newSpan(13, 14),
+						},
+						{
+							Kind:  TokenIdentifier,
+							Span:  newSpan(15, 16),
+							Value: "Z",
+						},
+						{
+							Kind: TokenRParen,
+							Span: newSpan(16, 17),
+						},
+					},
+				},
+				&CountOperator{
+					Pipe:    newSpan(18, 19),
+					Keyword: newSpan(20, 25),
+				},
+			},
+		},
+		err: true,
+	},
 }
 
 func TestParse(t *testing.T) {

From fb459dbd3f356417941344b9a98e3ab5941db606 Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 4 Jun 2024 14:33:30 -0700
Subject: [PATCH 04/16] Handle basic operator names

---
 autocomplete.go      | 107 ++++++++++++++----
 autocomplete_test.go | 259 +++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 336 insertions(+), 30 deletions(-)

diff --git a/autocomplete.go b/autocomplete.go
index 5461de4..957f046 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -6,6 +6,7 @@ package pql
 import (
 	"cmp"
 	"slices"
+	"sort"
 	"strings"
 
 	"github.com/runreveal/pql/parser"
@@ -24,7 +25,8 @@ type AnalysisColumn struct {
 }
 
 type Completion struct {
-	Identifier string
+	Label  string
+	Insert string
 }
 
 func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span) []*Completion {
@@ -35,34 +37,99 @@ func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span
 	expr, _ := parser.Parse(source)
 	if expr == nil {
 		prefix := completionPrefix(source, tokens, pos)
-		result := make([]*Completion, 0, len(ctx.Tables))
-		for tableName := range ctx.Tables {
-			if strings.HasPrefix(tableName, prefix) {
-				result = append(result, &Completion{
-					Identifier: tableName,
-				})
-			}
-		}
-		return result
+		return ctx.completeTableNames(prefix)
 	}
 
 	if sourceSpan := expr.Source.Span(); posSpan.Overlaps(sourceSpan) || pos < sourceSpan.Start {
 		// Assume that this is a table name.
 		prefix := completionPrefix(source, tokens, pos)
-		result := make([]*Completion, 0, len(ctx.Tables))
-		for tableName := range ctx.Tables {
-			if strings.HasPrefix(tableName, prefix) {
-				result = append(result, &Completion{
-					Identifier: tableName,
-				})
-			}
+		return ctx.completeTableNames(prefix)
+	}
+
+	// Find the operator that this cursor is associated with.
+	i := sort.Search(len(expr.Operators), func(i int) bool {
+		return expr.Operators[i].Span().Start >= pos
+	})
+	// Binary search will find the operator that follows the position.
+	// Since the first character is a pipe,
+	// we want to associate an exact match with the previous operator.
+	i--
+	if i < 0 {
+		// Before the first operator.
+		return completeOperators("")
+	}
+
+	switch op := expr.Operators[i].(type) {
+	case *parser.UnknownTabularOperator:
+		if pos <= op.Pipe.Start {
+			return completeOperators("")
+		}
+		if pos == op.Pipe.End {
+			return completeOperators("|")
+		}
+		if name := op.Name(); name != nil && name.NameSpan.Overlaps(posSpan) {
+			return completeOperators("| " + completionPrefix(source, tokens, pos))
+		}
+		if len(op.Tokens) == 0 || pos < op.Tokens[0].Span.Start {
+			return completeOperators("| ")
+		}
+		return nil
+	default:
+		return nil
+	}
+}
+
+var sortedOperatorNames = []string{
+	"as",
+	"count",
+	"extend",
+	"join",
+	"limit",
+	"order",
+	"project",
+	"sort",
+	"summarize",
+	"take",
+	"top",
+	"where",
+}
+
+func (ctx *AnalysisContext) completeTableNames(prefix string) []*Completion {
+	result := make([]*Completion, 0, len(ctx.Tables))
+	for tableName := range ctx.Tables {
+		if strings.HasPrefix(tableName, prefix) {
+			result = append(result, &Completion{
+				Label:  tableName,
+				Insert: tableName[len(prefix):],
+			})
 		}
-		return result
 	}
+	return result
+}
 
-	// TODO(now): More.
+func completeOperators(prefix string) []*Completion {
+	result := make([]*Completion, 0, len(sortedOperatorNames))
+	var namePrefix string
+	if rest, ok := strings.CutPrefix(prefix, "|"); ok {
+		if rest, ok = strings.CutPrefix(rest, " "); ok {
+			namePrefix = rest
+		}
+	}
 
-	return nil
+	for _, name := range sortedOperatorNames {
+		if !strings.HasPrefix(name, namePrefix) {
+			continue
+		}
+		c := &Completion{
+			Label:  name,
+			Insert: ("| " + name)[len(prefix):],
+		}
+		if name == "order" || name == "sort" {
+			c.Insert += " by"
+		}
+		result = append(result, c)
+	}
+	return result
 }
 
 func completionPrefix(source string, tokens []parser.Token, pos int) string {
diff --git a/autocomplete_test.go b/autocomplete_test.go
index 8913cc8..fd4d086 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -41,8 +41,8 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "",
 			sourceAfter:  "",
 			want: []*Completion{
-				{Identifier: "foo"},
-				{Identifier: "bar"},
+				{Label: "foo", Insert: "foo"},
+				{Label: "bar", Insert: "bar"},
 			},
 		},
 		{
@@ -65,7 +65,7 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "f",
 			sourceAfter:  "",
 			want: []*Completion{
-				{Identifier: "foo"},
+				{Label: "foo", Insert: "oo"},
 			},
 		},
 		{
@@ -88,8 +88,8 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "",
 			sourceAfter:  " | count",
 			want: []*Completion{
-				{Identifier: "foo"},
-				{Identifier: "bar"},
+				{Label: "foo", Insert: "foo"},
+				{Label: "bar", Insert: "bar"},
 			},
 		},
 		{
@@ -112,8 +112,8 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "",
 			sourceAfter:  "o | count",
 			want: []*Completion{
-				{Identifier: "foo"},
-				{Identifier: "bar"},
+				{Label: "foo", Insert: "foo"},
+				{Label: "bar", Insert: "bar"},
 			},
 		},
 		{
@@ -136,8 +136,244 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "",
 			sourceAfter:  " x | count",
 			want: []*Completion{
-				{Identifier: "foo"},
-				{Identifier: "bar"},
+				{Label: "foo", Insert: "foo"},
+				{Label: "bar", Insert: "bar"},
+			},
+		},
+		{
+			name: "FirstOperator",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "n"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "foo ",
+			sourceAfter:  "",
+			want: []*Completion{
+				{
+					Label:  "as",
+					Insert: "| as",
+				},
+				{
+					Label:  "count",
+					Insert: "| count",
+				},
+				{
+					Label:  "extend",
+					Insert: "| extend",
+				},
+				{
+					Label:  "join",
+					Insert: "| join",
+				},
+				{
+					Label:  "limit",
+					Insert: "| limit",
+				},
+				{
+					Label:  "order",
+					Insert: "| order by",
+				},
+				{
+					Label:  "project",
+					Insert: "| project",
+				},
+				{
+					Label:  "sort",
+					Insert: "| sort by",
+				},
+				{
+					Label:  "summarize",
+					Insert: "| summarize",
+				},
+				{
+					Label:  "take",
+					Insert: "| take",
+				},
+				{
+					Label:  "top",
+					Insert: "| top",
+				},
+				{
+					Label:  "where",
+					Insert: "| where",
+				},
+			},
+		},
+		{
+			name: "FirstOperatorAfterPipe",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "n"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "foo |",
+			sourceAfter:  "",
+			want: []*Completion{
+				{
+					Label:  "as",
+					Insert: " as",
+				},
+				{
+					Label:  "count",
+					Insert: " count",
+				},
+				{
+					Label:  "extend",
+					Insert: " extend",
+				},
+				{
+					Label:  "join",
+					Insert: " join",
+				},
+				{
+					Label:  "limit",
+					Insert: " limit",
+				},
+				{
+					Label:  "order",
+					Insert: " order by",
+				},
+				{
+					Label:  "project",
+					Insert: " project",
+				},
+				{
+					Label:  "sort",
+					Insert: " sort by",
+				},
+				{
+					Label:  "summarize",
+					Insert: " summarize",
+				},
+				{
+					Label:  "take",
+					Insert: " take",
+				},
+				{
+					Label:  "top",
+					Insert: " top",
+				},
+				{
+					Label:  "where",
+					Insert: " where",
+				},
+			},
+		},
+		{
+			name: "FirstOperatorAfterPipeSpace",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "n"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "foo |  ",
+			sourceAfter:  "",
+			want: []*Completion{
+				{
+					Label:  "as",
+					Insert: "as",
+				},
+				{
+					Label:  "count",
+					Insert: "count",
+				},
+				{
+					Label:  "extend",
+					Insert: "extend",
+				},
+				{
+					Label:  "join",
+					Insert: "join",
+				},
+				{
+					Label:  "limit",
+					Insert: "limit",
+				},
+				{
+					Label:  "order",
+					Insert: "order by",
+				},
+				{
+					Label:  "project",
+					Insert: "project",
+				},
+				{
+					Label:  "sort",
+					Insert: "sort by",
+				},
+				{
+					Label:  "summarize",
+					Insert: "summarize",
+				},
+				{
+					Label:  "take",
+					Insert: "take",
+				},
+				{
+					Label:  "top",
+					Insert: "top",
+				},
+				{
+					Label:  "where",
+					Insert: "where",
+				},
+			},
+		},
+		{
+			name: "FirstOperatorPartial",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "n"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "foo | whe",
+			sourceAfter:  "",
+			want: []*Completion{
+				{
+					Label:  "where",
+					Insert: "re",
+				},
 			},
 		},
 	}
@@ -149,7 +385,10 @@ func TestSuggestCompletions(t *testing.T) {
 				End:   len(test.sourceBefore),
 			})
 			completionLess := func(a, b *Completion) bool {
-				return a.Identifier < b.Identifier
+				if a.Label != b.Label {
+					return a.Label < b.Label
+				}
+				return a.Insert < b.Insert
 			}
 			if diff := cmp.Diff(test.want, got, cmpopts.SortSlices(completionLess)); diff != "" {
 				t.Errorf("SuggestCompletions(...) (-want +got):\n%s", diff)

From 7276675575c3808a6578fd1d122ef4f059780bfa Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 4 Jun 2024 15:29:41 -0700
Subject: [PATCH 05/16] Add basic column name completion

---
 autocomplete.go      | 73 ++++++++++++++++++++++++++++++++++++++++++--
 autocomplete_test.go | 26 +++++++++++++++-
 2 files changed, 95 insertions(+), 4 deletions(-)

diff --git a/autocomplete.go b/autocomplete.go
index 957f046..ad6699b 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -59,11 +59,10 @@ func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span
 		return completeOperators("")
 	}
 
+	columns := ctx.determineColumnsInScope(expr.Source, expr.Operators[:i])
+
 	switch op := expr.Operators[i].(type) {
 	case *parser.UnknownTabularOperator:
-		if pos <= op.Pipe.Start {
-			return completeOperators("")
-		}
 		if pos == op.Pipe.End {
 			return completeOperators("|")
 		}
@@ -74,6 +73,15 @@ func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span
 			return completeOperators("| ")
 		}
 		return nil
+	case *parser.WhereOperator:
+		if pos <= op.Keyword.Start {
+			return completeOperators("|")
+		}
+		if pos <= op.Keyword.End {
+			return nil
+		}
+		prefix := completionPrefix(source, tokens, pos)
+		return completeColumnNames(prefix, columns)
 	default:
 		return nil
 	}
@@ -94,6 +102,62 @@ var sortedOperatorNames = []string{
 	"where",
 }
 
+func (ctx *AnalysisContext) determineColumnsInScope(source parser.TabularDataSource, ops []parser.TabularOperator) []*AnalysisColumn {
+	var columns []*AnalysisColumn
+	if source, ok := source.(*parser.TableRef); ok {
+		columns = ctx.Tables[source.Table.Name].Columns
+	}
+	for _, op := range ops {
+		switch op := op.(type) {
+		case *parser.CountOperator:
+			columns = []*AnalysisColumn{{Name: "count()"}}
+		case *parser.ProjectOperator:
+			columns = make([]*AnalysisColumn, 0, len(op.Cols))
+			for _, col := range op.Cols {
+				columns = append(columns, &AnalysisColumn{
+					Name: col.Name.Name,
+				})
+			}
+		case *parser.ExtendOperator:
+			columns = slices.Clip(columns)
+			for _, col := range op.Cols {
+				columns = append(columns, &AnalysisColumn{
+					Name: col.Name.Name,
+				})
+			}
+		case *parser.SummarizeOperator:
+			columns = make([]*AnalysisColumn, 0, len(op.Cols)+len(op.GroupBy))
+			for _, col := range op.Cols {
+				columns = append(columns, &AnalysisColumn{
+					Name: col.Name.Name,
+				})
+			}
+			for _, col := range op.GroupBy {
+				columns = append(columns, &AnalysisColumn{
+					Name: col.Name.Name,
+				})
+			}
+		case *parser.JoinOperator:
+			columns = slices.Clip(columns)
+			columns = append(columns, ctx.determineColumnsInScope(op.Right.Source, op.Right.Operators)...)
+		}
+	}
+	return columns
+}
+
+func completeColumnNames(prefix string, columns []*AnalysisColumn) []*Completion {
+	result := make([]*Completion, 0, len(columns))
+	for _, col := range columns {
+		if strings.HasPrefix(col.Name, prefix) {
+			result = append(result, &Completion{
+				Label:  col.Name,
+				Insert: col.Name[len(prefix):],
+			})
+		}
+	}
+	return result
+}
+
 func (ctx *AnalysisContext) completeTableNames(prefix string) []*Completion {
 	result := make([]*Completion, 0, len(ctx.Tables))
 	for tableName := range ctx.Tables {
@@ -104,6 +168,9 @@ func (ctx *AnalysisContext) completeTableNames(prefix string) []*Completion {
 			})
 		}
 	}
+	slices.SortFunc(result, func(a, b *Completion) int {
+		return cmp.Compare(a.Label, b.Label)
+	})
 	return result
 }
 
diff --git a/autocomplete_test.go b/autocomplete_test.go
index fd4d086..9b35f3f 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -368,7 +368,6 @@ func TestSuggestCompletions(t *testing.T) {
 				},
 			},
 			sourceBefore: "foo | whe",
-			sourceAfter:  "",
 			want: []*Completion{
 				{
 					Label:  "where",
@@ -376,6 +375,31 @@ func TestSuggestCompletions(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "WhereExpression",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "name"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "foo | where n",
+			want: []*Completion{
+				{
+					Label:  "name",
+					Insert: "ame",
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {

From 95b9c8615dcf800d4ade565e60fc67781f1571db Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 4 Jun 2024 15:50:41 -0700
Subject: [PATCH 06/16] Fill out more operators for suggestions

---
 autocomplete.go      | 66 ++++++++++++++++++++++++++++++++++++++++----
 autocomplete_test.go | 50 +++++++++++++++++++++++++++++++++
 2 files changed, 111 insertions(+), 5 deletions(-)

diff --git a/autocomplete.go b/autocomplete.go
index ad6699b..1d8300d 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -31,18 +31,21 @@ type Completion struct {
 
 func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span) []*Completion {
 	pos := cursor.End
-	posSpan := parser.Span{Start: pos, End: pos}
 
 	tokens := parser.Scan(source)
 	expr, _ := parser.Parse(source)
+	prefix := completionPrefix(source, tokens, pos)
+	return ctx.suggestCompletions(expr, prefix, cursor.End)
+}
+
+func (ctx *AnalysisContext) suggestCompletions(expr *parser.TabularExpr, prefix string, pos int) []*Completion {
+	posSpan := parser.Span{Start: pos, End: pos}
 	if expr == nil {
-		prefix := completionPrefix(source, tokens, pos)
 		return ctx.completeTableNames(prefix)
 	}
 
 	if sourceSpan := expr.Source.Span(); posSpan.Overlaps(sourceSpan) || pos < sourceSpan.Start {
 		// Assume that this is a table name.
-		prefix := completionPrefix(source, tokens, pos)
 		return ctx.completeTableNames(prefix)
 	}
 
@@ -67,7 +70,7 @@ func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span
 			return completeOperators("|")
 		}
 		if name := op.Name(); name != nil && name.NameSpan.Overlaps(posSpan) {
-			return completeOperators("| " + completionPrefix(source, tokens, pos))
+			return completeOperators("| " + prefix)
 		}
 		if len(op.Tokens) == 0 || pos < op.Tokens[0].Span.Start {
 			return completeOperators("| ")
@@ -80,8 +83,61 @@ func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span
 		if pos <= op.Keyword.End {
 			return nil
 		}
-		prefix := completionPrefix(source, tokens, pos)
 		return completeColumnNames(prefix, columns)
+	case *parser.SortOperator:
+		if pos <= op.Keyword.Start {
+			return completeOperators("|")
+		}
+		if pos <= op.Keyword.End {
+			return nil
+		}
+		return completeColumnNames(prefix, columns)
+	case *parser.TopOperator:
+		if pos <= op.Keyword.Start {
+			return completeOperators("|")
+		}
+		if !op.By.IsValid() || pos <= op.By.End {
+			return nil
+		}
+		return completeColumnNames(prefix, columns)
+	case *parser.ProjectOperator:
+		if pos <= op.Keyword.Start {
+			return completeOperators("|")
+		}
+		if pos <= op.Keyword.End {
+			return nil
+		}
+		return completeColumnNames(prefix, columns)
+	case *parser.ExtendOperator:
+		if pos <= op.Keyword.Start {
+			return completeOperators("|")
+		}
+		if pos <= op.Keyword.End {
+			return nil
+		}
+		return completeColumnNames(prefix, columns)
+	case *parser.SummarizeOperator:
+		if pos <= op.Keyword.Start {
+			return completeOperators("|")
+		}
+		if pos <= op.Keyword.End {
+			return nil
+		}
+		return completeColumnNames(prefix, columns)
+	case *parser.JoinOperator:
+		if pos <= op.Keyword.Start {
+			return completeOperators("|")
+		}
+		if pos <= op.Keyword.End {
+			return nil
+		}
+		if op.Lparen.IsValid() && pos >= op.Lparen.End && (!op.Rparen.IsValid() || pos <= op.Rparen.Start) {
+			return ctx.suggestCompletions(op.Right, prefix, pos)
+		}
+		if op.On.IsValid() && pos > op.On.End {
+			return completeColumnNames(prefix, columns)
+		}
+		return nil
 	default:
 		return nil
 	}
diff --git a/autocomplete_test.go b/autocomplete_test.go
index 9b35f3f..7edb210 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -400,6 +400,56 @@ func TestSuggestCompletions(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "JoinExpression",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "name"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "foo | join (b",
+			want: []*Completion{
+				{
+					Label:  "bar",
+					Insert: "ar",
+				},
+			},
+		},
+		{
+			name: "JoinExpressionOn",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"foo": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+							{Name: "name"},
+						},
+					},
+					"bar": {
+						Columns: []*AnalysisColumn{
+							{Name: "id"},
+						},
+					},
+				},
+			},
+			sourceBefore: "foo | join (bar) on i",
+			want: []*Completion{
+				{
+					Label:  "id",
+					Insert: "d",
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {

From bee4cc422170833055c7974691f71bb637587a3c Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 4 Jun 2024 17:09:55 -0700
Subject: [PATCH 07/16] Add docs for autocomplete symbols

---
 autocomplete.go | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/autocomplete.go b/autocomplete.go
index 1d8300d..db9004a 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -12,23 +12,35 @@ import (
 	"github.com/runreveal/pql/parser"
 )
 
+// AnalysisContext is information about the eventual execution environment
+// passed in to assist in analysis tasks.
 type AnalysisContext struct {
 	Tables map[string]*AnalysisTable
 }
 
+// AnalysisTable is a table known to an [AnalysisContext].
 type AnalysisTable struct {
 	Columns []*AnalysisColumn
 }
 
+// AnalysisColumn is a column known to an [AnalysisTable].
 type AnalysisColumn struct {
 	Name string
 }
 
+// Completion is a single completion suggestion
+// returned by [AnalysisContext.SuggestCompletions].
 type Completion struct {
-	Label  string
+	// Label is the label that should be displayed for the completion.
+	// It represents the full string that is being completed.
+	Label string
+	// Insert is the text that should be inserted after the cursor
+	// to perform the completion.
 	Insert string
 }
 
+// SuggestCompletions suggests possible snippets to insert
+// given a partial pql statement and a selected range.
 func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span) []*Completion {
 	pos := cursor.End
 
@@ -161,7 +173,9 @@ var sortedOperatorNames = []string{
 func (ctx *AnalysisContext) determineColumnsInScope(source parser.TabularDataSource, ops []parser.TabularOperator) []*AnalysisColumn {
 	var columns []*AnalysisColumn
 	if source, ok := source.(*parser.TableRef); ok {
-		columns = ctx.Tables[source.Table.Name].Columns
+		if tab := ctx.Tables[source.Table.Name]; tab != nil {
+			columns = tab.Columns
+		}
 	}
 	for _, op := range ops {
 		switch op := op.(type) {

From b2ef7ccc3bdc3ca6ad9cabcdf01b15e7041d0feb Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Wed, 5 Jun 2024 08:29:41 -0700
Subject: [PATCH 08/16] Fix an autocomplete issue discovered through
 experimentation

---
 autocomplete.go       | 30 +++++++++++++++++++-----------
 autocomplete_test.go  | 21 +++++++++++++++++++++
 parser/parser_test.go | 19 +++++++++++++++++++
 3 files changed, 59 insertions(+), 11 deletions(-)

diff --git a/autocomplete.go b/autocomplete.go
index db9004a..5cfa092 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -6,7 +6,6 @@ package pql
 import (
 	"cmp"
 	"slices"
-	"sort"
 	"strings"
 
 	"github.com/runreveal/pql/parser"
@@ -62,13 +61,7 @@ func (ctx *AnalysisContext) suggestCompletions(expr *parser.TabularExpr, prefix
 	}
 
 	// Find the operator that this cursor is associated with.
-	i := sort.Search(len(expr.Operators), func(i int) bool {
-		return expr.Operators[i].Span().Start >= pos
-	})
-	// Binary search will find the operator that follows the position.
-	// Since the first character is a pipe,
-	// we want to associate an exact match with the previous operator.
-	i--
+	i := spanBefore(expr.Operators, pos, parser.TabularOperator.Span)
 	if i < 0 {
 		// Before the first operator.
 		return completeOperators("")
@@ -273,10 +266,12 @@ func completionPrefix(source string, tokens []parser.Token, pos int) string {
 	if len(tokens) == 0 {
 		return ""
 	}
-	i, _ := slices.BinarySearchFunc(tokens, pos, func(tok parser.Token, pos int) int {
-		return cmp.Compare(tok.Span.Start, pos)
+	i := spanBefore(tokens, pos, func(tok parser.Token) parser.Span {
+		return tok.Span
 	})
-	i = min(i, len(tokens)-1)
+	if i < 0 {
+		return ""
+	}
 	if !tokens[i].Span.Overlaps(parser.Span{Start: pos, End: pos}) || !isCompletableToken(tokens[i].Kind) {
 		// Cursor is not adjacent to token. Assume there's whitespace.
 		return ""
@@ -289,6 +284,19 @@ func completionPrefix(source string, tokens []parser.Token, pos int) string {
 	return source[start:pos]
 }
 
+// spanBefore finds the first span in a sorted slice
+// that starts before the given position.
+// The span function is used to obtain the span of each element in the slice.
+// If the position occurs before any spans,
+// spanBefore returns -1.
+func spanBefore[S ~[]E, E any](x S, pos int, span func(E) parser.Span) int {
+	i, _ := slices.BinarySearchFunc(x, pos, func(elem E, pos int) int {
+		return cmp.Compare(span(elem).Start, pos)
+	})
+	// Binary search will find the span that follows the position.
+	return i - 1
+}
+
 func isCompletableToken(kind parser.TokenKind) bool {
 	return kind == parser.TokenIdentifier ||
 		kind == parser.TokenQuotedIdentifier ||
diff --git a/autocomplete_test.go b/autocomplete_test.go
index 7edb210..b762953 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -450,6 +450,27 @@ func TestSuggestCompletions(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "Project",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"People": {
+						Columns: []*AnalysisColumn{
+							{Name: "FirstName"},
+							{Name: "LastName"},
+						},
+					},
+				},
+			},
+			sourceBefore: "People\n| project F",
+			sourceAfter:  ", LastName",
+			want: []*Completion{
+				{
+					Label:  "FirstName",
+					Insert: "irstName",
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
diff --git a/parser/parser_test.go b/parser/parser_test.go
index 90f56a2..15a65a8 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -1895,6 +1895,25 @@ var parserTests = []struct {
 		},
 		err: true,
 	},
+	{
+		name:  "PartialProject",
+		query: "People | project , LastName",
+		want: &TabularExpr{
+			Source: &TableRef{
+				Table: &Ident{
+					Name:     "People",
+					NameSpan: newSpan(0, 6),
+				},
+			},
+			Operators: []TabularOperator{
+				&ProjectOperator{
+					Pipe:    newSpan(7, 8),
+					Keyword: newSpan(9, 16),
+				},
+			},
+		},
+		err: true,
+	},
 }
 
 func TestParse(t *testing.T) {

From e988458a1669269701f16367c0c4e0399806e0bf Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Wed, 5 Jun 2024 09:37:23 -0700
Subject: [PATCH 09/16] Check in testing UI

---
 cmd/pql-playground/app.js     |  68 ++++++++++++++++++
 cmd/pql-playground/index.html |  56 +++++++++++++++
 cmd/pql-playground/main.go    | 126 ++++++++++++++++++++++++++++++++++
 3 files changed, 250 insertions(+)
 create mode 100644 cmd/pql-playground/app.js
 create mode 100644 cmd/pql-playground/index.html
 create mode 100644 cmd/pql-playground/main.go

diff --git a/cmd/pql-playground/app.js b/cmd/pql-playground/app.js
new file mode 100644
index 0000000..995bbb8
--- /dev/null
+++ b/cmd/pql-playground/app.js
@@ -0,0 +1,68 @@
+import { Application, Controller } from 'https://unpkg.com/@hotwired/stimulus/dist/stimulus.js';
+
+window.Stimulus = Application.start();
+Stimulus.register('analysis', class extends Controller {
+  static targets = ['list', 'editor', 'output'];
+  static values = {
+    compileHref: String,
+    suggestHref: String,
+  };
+
+  /**
+   * @param {Event} event
+   * @return {Promise<void>}
+   */
+  async compile(event) {
+    const formData = new URLSearchParams();
+    formData.append('source', this.editorTarget.value);
+    const response = await fetch(this.compileHrefValue, {
+      method: 'POST',
+      body: formData,
+    });
+    if (!response.ok) {
+      return;
+    }
+    this.outputTarget.innerHTML = await response.text();
+    this.outputTarget.hidden = false;
+  }
+
+  /**
+   * @param {Event} event
+   * @return {Promise<void>}
+   */
+  async suggest(event) {
+    const formData = new URLSearchParams();
+    formData.append('source', this.editorTarget.value);
+    formData.append('start', this.editorTarget.selectionStart);
+    formData.append('end', this.editorTarget.selectionEnd);
+    const response = await fetch(this.suggestHrefValue, {
+      method: 'POST',
+      body: formData,
+    });
+    if (!response.ok) {
+      return;
+    }
+    this.listTarget.innerHTML = await response.text();
+    this.listTarget.hidden = false;
+  }
+
+  clear() {
+    this.listTarget.innerHTML = '';
+    this.listTarget.hidden = true;
+    this.editorTarget.focus();
+  }
+
+  /**
+   * @param {Event} event
+   */
+  fill(event) {
+    const i = this.editorTarget.selectionEnd;
+    this.editorTarget.value = this.editorTarget.value.substring(0, i) +
+      event.params.insert +
+      this.editorTarget.value.substring(i);
+    const insertEnd = i + event.params.insert.length;
+    this.editorTarget.setSelectionRange(insertEnd, insertEnd);
+
+    this.clear();
+  }
+});
diff --git a/cmd/pql-playground/index.html b/cmd/pql-playground/index.html
new file mode 100644
index 0000000..c119617
--- /dev/null
+++ b/cmd/pql-playground/index.html
@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>pql Autocomplete Playground</title>
+    <script
+      type="module"
+      src="https://unpkg.com/@hotwired/stimulus/dist/stimulus.js"
+    ></script>
+    <script type="module" src="/app.js"></script>
+    <script src="https://cdn.tailwindcss.com"></script>
+  </head>
+  <body class="bg-slate-200">
+    <div
+      class="p-16"
+      data-controller="analysis"
+      data-analysis-compile-href-value="/compile"
+      data-analysis-suggest-href-value="/suggest"
+    >
+      <form
+        data-action="analysis#compile:prevent keydown.ctrl+enter->analysis#compile"
+        class="mx-auto my-8 grid grid-cols-[2fr_1fr] max-w-[90ch] gap-x-4 gap-y-8"
+      >
+        <div>
+          <textarea
+            name="source"
+            rows="20"
+            class="p-2 border border-slate-400 font-mono w-full bg-slate-50"
+            autofocus
+            data-analysis-target="editor"
+            data-action="input->analysis#clear selectionchange->analysis#clear keydown.ctrl+space->analysis#suggest keydown.esc->analysis#clear"
+          ></textarea>
+        </div>
+
+        <div>
+          <ul
+            class="bg-slate-700 text-slate-100"
+            data-analysis-target="list"
+            data-action="keydown.esc->analysis#clear"
+            hidden
+          ></ul>
+        </div>
+
+        <div class="text-center">
+          <input
+            type="submit"
+            value="Compile"
+            class="inline-block px-8 py-4 text-lg font-bold bg-blue-600 text-white rounded-md outline-none focus:ring focus:ring-orange-400"
+          />
+        </div>
+      </form>
+
+      <div class="mx-auto my-16 border-t border-slate-400 p-8 max-w-[80ch]" data-analysis-target="output" hidden></div>
+    </div>
+  </body>
+</html>
diff --git a/cmd/pql-playground/main.go b/cmd/pql-playground/main.go
new file mode 100644
index 0000000..0053a85
--- /dev/null
+++ b/cmd/pql-playground/main.go
@@ -0,0 +1,126 @@
+package main
+
+import (
+	"bytes"
+	"embed"
+	"errors"
+	"html"
+	"io"
+	"io/fs"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/runreveal/pql"
+	"github.com/runreveal/pql/parser"
+)
+
+//go:embed index.html
+//go:embed app.js
+var static embed.FS
+
+func main() {
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		serveFileFS(w, r, static, "index.html")
+	})
+	http.HandleFunc("/app.js", func(w http.ResponseWriter, r *http.Request) {
+		serveFileFS(w, r, static, "app.js")
+	})
+	http.HandleFunc("/compile", compile)
+	http.HandleFunc("/suggest", suggest)
+
+	log.Println("Listening")
+	http.ListenAndServe(":8080", nil)
+}
+
+func compile(w http.ResponseWriter, r *http.Request) {
+	sql, err := pql.Compile(r.FormValue("source"))
+	buf := new(bytes.Buffer)
+	if err != nil {
+		buf.WriteString(`<div class="italic text-red-900">`)
+		buf.WriteString(html.EscapeString(err.Error()))
+		buf.WriteString("</div>")
+	} else {
+		buf.WriteString(`<div class="font-mono bg-slate-300 text-black p-4">`)
+		for _, line := range strings.Split(sql, "\n") {
+			buf.WriteString(`<pre class="whitespace-pre-line ps-8 -indent-8"><code>`)
+			buf.WriteString(html.EscapeString(line))
+			buf.WriteString("</code></pre>\n")
+		}
+		buf.WriteString("</div>\n")
+	}
+	writeBuffer(w, buf)
+}
+
+func suggest(w http.ResponseWriter, r *http.Request) {
+	ctx := &pql.AnalysisContext{
+		Tables: map[string]*pql.AnalysisTable{
+			"People": {
+				Columns: []*pql.AnalysisColumn{
+					{Name: "FirstName"},
+					{Name: "LastName"},
+					{Name: "PhoneNumber"},
+				},
+			},
+		},
+	}
+	start, err := strconv.Atoi(r.FormValue("start"))
+	if err != nil {
+		http.Error(w, "start: "+err.Error(), http.StatusUnprocessableEntity)
+		return
+	}
+	end, err := strconv.Atoi(r.FormValue("end"))
+	if err != nil {
+		http.Error(w, "end: "+err.Error(), http.StatusUnprocessableEntity)
+		return
+	}
+	completions := ctx.SuggestCompletions(r.FormValue("source"), parser.Span{
+		Start: start,
+		End:   end,
+	})
+
+	buf := new(bytes.Buffer)
+	if len(completions) == 0 {
+		buf.WriteString(`<li class="p-2">No completions.</li>`)
+	} else {
+		for _, c := range completions {
+			buf.WriteString(`<li class="p-2 hover:bg-white/25 has-[:focus]:bg-white/50 has-[:active]:bg-white/50"><a class="outline-none" href="#" data-action="analysis#fill" data-analysis-insert-param="`)
+			buf.WriteString(html.EscapeString(c.Insert))
+			buf.WriteString(`">`)
+			buf.WriteString(html.EscapeString(c.Label))
+			buf.WriteString("</a></li>\n")
+		}
+	}
+	writeBuffer(w, buf)
+}
+
+func writeBuffer(w http.ResponseWriter, buf *bytes.Buffer) {
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
+	io.Copy(w, buf)
+}
+
+func serveFileFS(w http.ResponseWriter, r *http.Request, fsys fs.FS, name string) {
+	f, err := fsys.Open(name)
+	if errors.Is(err, fs.ErrNotExist) {
+		http.NotFound(w, r)
+		return
+	}
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	defer f.Close()
+	content, ok := f.(io.ReadSeeker)
+	if !ok {
+		contentBytes, err := io.ReadAll(f)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		content = bytes.NewReader(contentBytes)
+	}
+	http.ServeContent(w, r, name, time.Time{}, content)
+}

From 0f8fcc3d8515eaf25ac5aa5417553c5b3334d710 Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 11 Jun 2024 13:20:46 -0700
Subject: [PATCH 10/16] Change API to support replacements

---
 autocomplete.go            | 197 ++++++++++--------
 autocomplete_test.go       | 415 +++++++++++++++++++++++++++++--------
 cmd/pql-playground/app.js  |  11 +-
 cmd/pql-playground/main.go |   8 +-
 4 files changed, 445 insertions(+), 186 deletions(-)

diff --git a/autocomplete.go b/autocomplete.go
index 748a7c8..6912e59 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -31,11 +31,16 @@ type AnalysisColumn struct {
 // returned by [AnalysisContext.SuggestCompletions].
 type Completion struct {
 	// Label is the label that should be displayed for the completion.
-	// It represents the full string that is being completed.
+	// It is often the same as Text,
+	// but Text may include extra characters for convenience.
 	Label string
-	// Insert is the text that should be inserted after the cursor
-	// to perform the completion.
-	Insert string
+	// Text is the full string that is being placed.
+	Text string
+	// Span is the position where Text should be placed.
+	// If the span's length is zero,
+	// then the text should be inserted for a successful completion.
+	// Otherwise, the span indicates text that should be replaced with the text.
+	Span parser.Span
 }
 
 // SuggestCompletions suggests possible snippets to insert
@@ -45,10 +50,10 @@ func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span
 
 	tokens := parser.Scan(source)
 	stmts, _ := parser.Parse(source)
-	prefix := completionPrefix(source, tokens, pos)
+	prefix := completionPrefix(tokens, pos)
 	i := spanBefore(stmts, pos, parser.Statement.Span)
 	if i < 0 {
-		return ctx.completeTableNames(prefix)
+		return ctx.completeTableNames(source, prefix)
 	}
 	letNames := make(map[string]struct{})
 	for _, stmt := range stmts[:i] {
@@ -58,113 +63,112 @@ func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span
 	}
 	switch stmt := stmts[i].(type) {
 	case *parser.LetStatement:
-		return ctx.suggestLetStatement(stmt, letNames, prefix, pos)
+		return ctx.suggestLetStatement(source, stmt, letNames, prefix)
 	case *parser.TabularExpr:
-		return ctx.suggestTabularExpr(stmt, letNames, prefix, pos)
+		return ctx.suggestTabularExpr(source, stmt, letNames, prefix)
 	default:
 		return nil
 	}
 }
 
-func (ctx *AnalysisContext) suggestLetStatement(stmt *parser.LetStatement, letNames map[string]struct{}, prefix string, pos int) []*Completion {
-	if !stmt.Assign.IsValid() || pos < stmt.Assign.End {
+func (ctx *AnalysisContext) suggestLetStatement(source string, stmt *parser.LetStatement, letNames map[string]struct{}, prefix parser.Span) []*Completion {
+	if !stmt.Assign.IsValid() || prefix.End < stmt.Assign.End {
 		return nil
 	}
-	return completeScope(prefix, letNames)
+	return completeScope(source, prefix, letNames)
 }
 
-func (ctx *AnalysisContext) suggestTabularExpr(expr *parser.TabularExpr, letNames map[string]struct{}, prefix string, pos int) []*Completion {
-	posSpan := parser.Span{Start: pos, End: pos}
+func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.TabularExpr, letNames map[string]struct{}, prefix parser.Span) []*Completion {
 	if expr == nil {
-		return ctx.completeTableNames(prefix)
+		return ctx.completeTableNames(source, prefix)
 	}
 
-	if sourceSpan := expr.Source.Span(); posSpan.Overlaps(sourceSpan) || pos < sourceSpan.Start {
+	if sourceSpan := expr.Source.Span(); prefix.Overlaps(sourceSpan) || prefix.End < sourceSpan.Start {
 		// Assume that this is a table name.
-		return ctx.completeTableNames(prefix)
+		return ctx.completeTableNames(source, prefix)
 	}
 
 	// Find the operator that this cursor is associated with.
-	i := spanBefore(expr.Operators, pos, parser.TabularOperator.Span)
+	i := spanBefore(expr.Operators, prefix.End, parser.TabularOperator.Span)
 	if i < 0 {
 		// Before the first operator.
-		return completeOperators("")
+		return completeOperators(source, prefix, true)
 	}
 
 	columns := ctx.determineColumnsInScope(expr.Source, expr.Operators[:i])
 
 	switch op := expr.Operators[i].(type) {
 	case *parser.UnknownTabularOperator:
-		if pos == op.Pipe.End {
-			return completeOperators("|")
+		if prefix.End == op.Pipe.End {
+			return completeOperators(source, prefix, false)
 		}
-		if name := op.Name(); name != nil && name.NameSpan.Overlaps(posSpan) {
-			return completeOperators("| " + prefix)
+		if name := op.Name(); name != nil && name.NameSpan.Overlaps(prefix) {
+			return completeOperators(source, prefix, false)
 		}
-		if len(op.Tokens) == 0 || pos < op.Tokens[0].Span.Start {
-			return completeOperators("| ")
+		if len(op.Tokens) == 0 || prefix.End < op.Tokens[0].Span.Start {
+			return completeOperators(source, prefix, false)
 		}
 		return nil
 	case *parser.WhereOperator:
-		if pos <= op.Keyword.Start {
-			return completeOperators("|")
+		if prefix.End <= op.Keyword.Start {
+			return completeOperators(source, prefix, false)
 		}
-		if pos <= op.Keyword.End {
+		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		return completeColumnNames(prefix, columns)
+		return completeColumnNames(source, prefix, columns)
 	case *parser.SortOperator:
-		if pos <= op.Keyword.Start {
-			return completeOperators("|")
+		if prefix.End <= op.Keyword.Start {
+			return completeOperators(source, prefix, false)
 		}
-		if pos <= op.Keyword.End {
+		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		return completeColumnNames(prefix, columns)
+		return completeColumnNames(source, prefix, columns)
 	case *parser.TopOperator:
-		if pos <= op.Keyword.Start {
-			return completeOperators("|")
+		if prefix.End <= op.Keyword.Start {
+			return completeOperators(source, prefix, false)
 		}
-		if !op.By.IsValid() || pos <= op.By.End {
+		if !op.By.IsValid() || prefix.End <= op.By.End {
 			return nil
 		}
-		return completeColumnNames(prefix, columns)
+		return completeColumnNames(source, prefix, columns)
 	case *parser.ProjectOperator:
-		if pos <= op.Keyword.Start {
-			return completeOperators("|")
+		if prefix.End <= op.Keyword.Start {
+			return completeOperators(source, prefix, false)
 		}
-		if pos <= op.Keyword.End {
+		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		return completeColumnNames(prefix, columns)
+		return completeColumnNames(source, prefix, columns)
 	case *parser.ExtendOperator:
-		if pos <= op.Keyword.Start {
-			return completeOperators("|")
+		if prefix.End <= op.Keyword.Start {
+			return completeOperators(source, prefix, false)
 		}
-		if pos <= op.Keyword.End {
+		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		return completeColumnNames(prefix, columns)
+		return completeColumnNames(source, prefix, columns)
 	case *parser.SummarizeOperator:
-		if pos <= op.Keyword.Start {
-			return completeOperators("|")
+		if prefix.End <= op.Keyword.Start {
+			return completeOperators(source, prefix, false)
 		}
-		if pos <= op.Keyword.End {
+		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		return completeColumnNames(prefix, columns)
+		return completeColumnNames(source, prefix, columns)
 	case *parser.JoinOperator:
-		if pos <= op.Keyword.Start {
-			return completeOperators("|")
+		if prefix.End <= op.Keyword.Start {
+			return completeOperators(source, prefix, false)
 		}
-		if pos <= op.Keyword.End {
+		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		if op.Lparen.IsValid() && pos >= op.Lparen.End && (!op.Rparen.IsValid() || pos <= op.Rparen.Start) {
-			return ctx.suggestTabularExpr(op.Right, letNames, prefix, pos)
+		if op.Lparen.IsValid() && prefix.End >= op.Lparen.End && (!op.Rparen.IsValid() || prefix.End <= op.Rparen.Start) {
+			return ctx.suggestTabularExpr(source, op.Right, letNames, prefix)
 		}
-		if op.On.IsValid() && pos > op.On.End {
-			return completeColumnNames(prefix, columns)
+		if op.On.IsValid() && prefix.End > op.On.End {
+			return completeColumnNames(source, prefix, columns)
 		}
 		return nil
 	default:
@@ -232,93 +236,116 @@ func (ctx *AnalysisContext) determineColumnsInScope(source parser.TabularDataSou
 	return columns
 }
 
-func completeColumnNames(prefix string, columns []*AnalysisColumn) []*Completion {
+func completeColumnNames(source string, prefixSpan parser.Span, columns []*AnalysisColumn) []*Completion {
+	prefix := source[prefixSpan.Start:prefixSpan.End]
+
 	result := make([]*Completion, 0, len(columns))
 	for _, col := range columns {
 		if strings.HasPrefix(col.Name, prefix) {
 			result = append(result, &Completion{
-				Label:  col.Name,
-				Insert: col.Name[len(prefix):],
+				Label: col.Name,
+				Text:  col.Name,
+				Span:  prefixSpan,
 			})
 		}
 	}
 	return result
 }
 
-func completeScope(prefix string, scope map[string]struct{}) []*Completion {
+func completeScope(source string, prefixSpan parser.Span, scope map[string]struct{}) []*Completion {
+	prefix := source[prefixSpan.Start:prefixSpan.End]
+
 	result := make([]*Completion, 0, len(scope))
 	for name := range scope {
 		if strings.HasPrefix(name, prefix) {
 			result = append(result, &Completion{
-				Label:  name,
-				Insert: name[len(prefix):],
+				Label: name,
+				Text:  name,
+				Span:  prefixSpan,
 			})
 		}
 	}
 	return result
 }
 
-func (ctx *AnalysisContext) completeTableNames(prefix string) []*Completion {
+func (ctx *AnalysisContext) completeTableNames(source string, prefixSpan parser.Span) []*Completion {
+	prefix := source[prefixSpan.Start:prefixSpan.End]
+
 	result := make([]*Completion, 0, len(ctx.Tables))
 	for tableName := range ctx.Tables {
 		if strings.HasPrefix(tableName, prefix) {
 			result = append(result, &Completion{
-				Label:  tableName,
-				Insert: tableName[len(prefix):],
+				Label: tableName,
+				Text:  tableName,
+				Span:  prefixSpan,
 			})
 		}
 	}
 	slices.SortFunc(result, func(a, b *Completion) int {
-		return cmp.Compare(a.Label, b.Label)
+		return cmp.Compare(a.Text, b.Text)
 	})
 	return result
 }
 
-func completeOperators(prefix string) []*Completion {
-	result := make([]*Completion, 0, len(sortedOperatorNames))
-	var namePrefix string
-	if rest, ok := strings.CutPrefix(prefix, "|"); ok {
-		if rest, ok = strings.CutPrefix(rest, " "); ok {
-			namePrefix = rest
+func completeOperators(source string, prefixSpan parser.Span, includePipe bool) []*Completion {
+	if includePipe {
+		// Should always be an insert, not a replacement.
+		prefixSpan = parser.Span{
+			Start: prefixSpan.End,
+			End:   prefixSpan.End,
 		}
 	}
+	prefix := source[prefixSpan.Start:prefixSpan.End]
+	leading := ""
+	if includePipe {
+		leading = "| "
+	} else if prefixSpan.Len() == 0 && prefixSpan.Start > 0 && source[prefixSpan.Start-1] == '|' {
+		// If directly adjacent to pipe, automatically add a space.
+		leading = " "
+	}
 
+	result := make([]*Completion, 0, len(sortedOperatorNames))
 	for _, name := range sortedOperatorNames {
-		if !strings.HasPrefix(name, namePrefix) {
+		if !strings.HasPrefix(name, prefix) {
 			continue
 		}
 		c := &Completion{
-			Label:  name,
-			Insert: ("| " + name)[len(prefix):],
+			Label: name,
+			Text:  leading + name,
+			Span:  prefixSpan,
 		}
 		if name == "order" || name == "sort" {
-			c.Insert += " by"
+			c.Text += " by"
 		}
 		result = append(result, c)
 	}
 	return result
 }
 
-func completionPrefix(source string, tokens []parser.Token, pos int) string {
+// completionPrefix returns a span of characters that should be considered for
+// filtering completion results and replacement during the completion.
+func completionPrefix(tokens []parser.Token, pos int) parser.Span {
+	result := parser.Span{
+		Start: pos,
+		End:   pos,
+	}
 	if len(tokens) == 0 {
-		return ""
+		return result
 	}
-	i := spanBefore(tokens, pos, func(tok parser.Token) parser.Span {
-		return tok.Span
-	})
+	i := spanBefore(tokens, pos, func(tok parser.Token) parser.Span { return tok.Span })
 	if i < 0 {
-		return ""
+		return result
 	}
 	if !tokens[i].Span.Overlaps(parser.Span{Start: pos, End: pos}) || !isCompletableToken(tokens[i].Kind) {
 		// Cursor is not adjacent to token. Assume there's whitespace.
-		return ""
+		return result
 	}
-	start := tokens[i].Span.Start
+	result.Start = tokens[i].Span.Start
 	if tokens[i].Kind == parser.TokenQuotedIdentifier {
 		// Skip past initial backtick.
-		start += len("`")
+		result.Start += len("`")
 	}
-	return source[start:pos]
+	return result
 }
 
 // spanBefore finds the first span in a sorted slice
diff --git a/autocomplete_test.go b/autocomplete_test.go
index b762953..95f1a48 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -41,8 +41,22 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "",
 			sourceAfter:  "",
 			want: []*Completion{
-				{Label: "foo", Insert: "foo"},
-				{Label: "bar", Insert: "bar"},
+				{
+					Label: "foo",
+					Text:  "foo",
+					Span: parser.Span{
+						Start: 0,
+						End:   0,
+					},
+				},
+				{
+					Label: "bar",
+					Text:  "bar",
+					Span: parser.Span{
+						Start: 0,
+						End:   0,
+					},
+				},
 			},
 		},
 		{
@@ -65,7 +79,14 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "f",
 			sourceAfter:  "",
 			want: []*Completion{
-				{Label: "foo", Insert: "oo"},
+				{
+					Label: "foo",
+					Text:  "foo",
+					Span: parser.Span{
+						Start: 0,
+						End:   1,
+					},
+				},
 			},
 		},
 		{
@@ -88,8 +109,22 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "",
 			sourceAfter:  " | count",
 			want: []*Completion{
-				{Label: "foo", Insert: "foo"},
-				{Label: "bar", Insert: "bar"},
+				{
+					Label: "foo",
+					Text:  "foo",
+					Span: parser.Span{
+						Start: 0,
+						End:   0,
+					},
+				},
+				{
+					Label: "bar",
+					Text:  "bar",
+					Span: parser.Span{
+						Start: 0,
+						End:   0,
+					},
+				},
 			},
 		},
 		{
@@ -112,8 +147,22 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "",
 			sourceAfter:  "o | count",
 			want: []*Completion{
-				{Label: "foo", Insert: "foo"},
-				{Label: "bar", Insert: "bar"},
+				{
+					Label: "foo",
+					Text:  "foo",
+					Span: parser.Span{
+						Start: 0,
+						End:   0,
+					},
+				},
+				{
+					Label: "bar",
+					Text:  "bar",
+					Span: parser.Span{
+						Start: 0,
+						End:   0,
+					},
+				},
 			},
 		},
 		{
@@ -136,8 +185,20 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "",
 			sourceAfter:  " x | count",
 			want: []*Completion{
-				{Label: "foo", Insert: "foo"},
-				{Label: "bar", Insert: "bar"},
+				{
+					Label: "foo",
+					Text:  "foo",
+					Span: parser.Span{
+						Start: 0,
+						End:   0,
+					}},
+				{
+					Label: "bar",
+					Text:  "bar",
+					Span: parser.Span{
+						Start: 0,
+						End:   0,
+					}},
 			},
 		},
 		{
@@ -161,52 +222,100 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceAfter:  "",
 			want: []*Completion{
 				{
-					Label:  "as",
-					Insert: "| as",
+					Label: "as",
+					Text:  "| as",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "count",
-					Insert: "| count",
+					Label: "count",
+					Text:  "| count",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "extend",
-					Insert: "| extend",
+					Label: "extend",
+					Text:  "| extend",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "join",
-					Insert: "| join",
+					Label: "join",
+					Text:  "| join",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "limit",
-					Insert: "| limit",
+					Label: "limit",
+					Text:  "| limit",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "order",
-					Insert: "| order by",
+					Label: "order",
+					Text:  "| order by",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "project",
-					Insert: "| project",
+					Label: "project",
+					Text:  "| project",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "sort",
-					Insert: "| sort by",
+					Label: "sort",
+					Text:  "| sort by",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "summarize",
-					Insert: "| summarize",
+					Label: "summarize",
+					Text:  "| summarize",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "take",
-					Insert: "| take",
+					Label: "take",
+					Text:  "| take",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "top",
-					Insert: "| top",
+					Label: "top",
+					Text:  "| top",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 				{
-					Label:  "where",
-					Insert: "| where",
+					Label: "where",
+					Text:  "| where",
+					Span: parser.Span{
+						Start: 4,
+						End:   4,
+					},
 				},
 			},
 		},
@@ -231,52 +340,100 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceAfter:  "",
 			want: []*Completion{
 				{
-					Label:  "as",
-					Insert: " as",
+					Label: "as",
+					Text:  " as",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "count",
-					Insert: " count",
+					Label: "count",
+					Text:  " count",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "extend",
-					Insert: " extend",
+					Label: "extend",
+					Text:  " extend",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "join",
-					Insert: " join",
+					Label: "join",
+					Text:  " join",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "limit",
-					Insert: " limit",
+					Label: "limit",
+					Text:  " limit",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "order",
-					Insert: " order by",
+					Label: "order",
+					Text:  " order by",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "project",
-					Insert: " project",
+					Label: "project",
+					Text:  " project",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "sort",
-					Insert: " sort by",
+					Label: "sort",
+					Text:  " sort by",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "summarize",
-					Insert: " summarize",
+					Label: "summarize",
+					Text:  " summarize",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "take",
-					Insert: " take",
+					Label: "take",
+					Text:  " take",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "top",
-					Insert: " top",
+					Label: "top",
+					Text:  " top",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 				{
-					Label:  "where",
-					Insert: " where",
+					Label: "where",
+					Text:  " where",
+					Span: parser.Span{
+						Start: 5,
+						End:   5,
+					},
 				},
 			},
 		},
@@ -301,52 +458,100 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceAfter:  "",
 			want: []*Completion{
 				{
-					Label:  "as",
-					Insert: "as",
+					Label: "as",
+					Text:  "as",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "count",
-					Insert: "count",
+					Label: "count",
+					Text:  "count",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "extend",
-					Insert: "extend",
+					Label: "extend",
+					Text:  "extend",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "join",
-					Insert: "join",
+					Label: "join",
+					Text:  "join",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "limit",
-					Insert: "limit",
+					Label: "limit",
+					Text:  "limit",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "order",
-					Insert: "order by",
+					Label: "order",
+					Text:  "order by",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "project",
-					Insert: "project",
+					Label: "project",
+					Text:  "project",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "sort",
-					Insert: "sort by",
+					Label: "sort",
+					Text:  "sort by",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "summarize",
-					Insert: "summarize",
+					Label: "summarize",
+					Text:  "summarize",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "take",
-					Insert: "take",
+					Label: "take",
+					Text:  "take",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "top",
-					Insert: "top",
+					Label: "top",
+					Text:  "top",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 				{
-					Label:  "where",
-					Insert: "where",
+					Label: "where",
+					Text:  "where",
+					Span: parser.Span{
+						Start: 7,
+						End:   7,
+					},
 				},
 			},
 		},
@@ -370,8 +575,12 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "foo | whe",
 			want: []*Completion{
 				{
-					Label:  "where",
-					Insert: "re",
+					Label: "where",
+					Text:  "where",
+					Span: parser.Span{
+						Start: 6,
+						End:   9,
+					},
 				},
 			},
 		},
@@ -395,8 +604,12 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "foo | where n",
 			want: []*Completion{
 				{
-					Label:  "name",
-					Insert: "ame",
+					Label: "name",
+					Text:  "name",
+					Span: parser.Span{
+						Start: 12,
+						End:   13,
+					},
 				},
 			},
 		},
@@ -420,8 +633,12 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "foo | join (b",
 			want: []*Completion{
 				{
-					Label:  "bar",
-					Insert: "ar",
+					Label: "bar",
+					Text:  "bar",
+					Span: parser.Span{
+						Start: 12,
+						End:   13,
+					},
 				},
 			},
 		},
@@ -445,8 +662,12 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceBefore: "foo | join (bar) on i",
 			want: []*Completion{
 				{
-					Label:  "id",
-					Insert: "d",
+					Label: "id",
+					Text:  "id",
+					Span: parser.Span{
+						Start: 20,
+						End:   21,
+					},
 				},
 			},
 		},
@@ -466,8 +687,12 @@ func TestSuggestCompletions(t *testing.T) {
 			sourceAfter:  ", LastName",
 			want: []*Completion{
 				{
-					Label:  "FirstName",
-					Insert: "irstName",
+					Label: "FirstName",
+					Text:  "FirstName",
+					Span: parser.Span{
+						Start: 17,
+						End:   18,
+					},
 				},
 			},
 		},
@@ -480,10 +705,16 @@ func TestSuggestCompletions(t *testing.T) {
 				End:   len(test.sourceBefore),
 			})
 			completionLess := func(a, b *Completion) bool {
+				if a.Span.Start != b.Span.Start {
+					return a.Span.Start < b.Span.Start
+				}
+				if a.Span.End != b.Span.End {
+					return a.Span.End < b.Span.End
+				}
 				if a.Label != b.Label {
 					return a.Label < b.Label
 				}
-				return a.Insert < b.Insert
+				return a.Text < b.Text
 			}
 			if diff := cmp.Diff(test.want, got, cmpopts.SortSlices(completionLess)); diff != "" {
 				t.Errorf("SuggestCompletions(...) (-want +got):\n%s", diff)
diff --git a/cmd/pql-playground/app.js b/cmd/pql-playground/app.js
index 995bbb8..01d8105 100644
--- a/cmd/pql-playground/app.js
+++ b/cmd/pql-playground/app.js
@@ -56,12 +56,11 @@ Stimulus.register('analysis', class extends Controller {
    * @param {Event} event
    */
   fill(event) {
-    const i = this.editorTarget.selectionEnd;
-    this.editorTarget.value = this.editorTarget.value.substring(0, i) +
-      event.params.insert +
-      this.editorTarget.value.substring(i);
-    const insertEnd = i + event.params.insert.length;
-    this.editorTarget.setSelectionRange(insertEnd, insertEnd);
+    this.editorTarget.value = this.editorTarget.value.substring(0, event.params.start) +
+      event.params.text +
+      this.editorTarget.value.substring(event.params.end);
+    const i = event.params.start + event.params.insert.length;
+    this.editorTarget.setSelectionRange(i, i);
 
     this.clear();
   }
diff --git a/cmd/pql-playground/main.go b/cmd/pql-playground/main.go
index 0053a85..6beb574 100644
--- a/cmd/pql-playground/main.go
+++ b/cmd/pql-playground/main.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"embed"
 	"errors"
+	"fmt"
 	"html"
 	"io"
 	"io/fs"
@@ -86,9 +87,10 @@ func suggest(w http.ResponseWriter, r *http.Request) {
 		buf.WriteString(`<li class="p-2">No completions.</li>`)
 	} else {
 		for _, c := range completions {
-			buf.WriteString(`<li class="p-2 hover:bg-white/25 has-[:focus]:bg-white/50 has-[:active]:bg-white/50"><a class="outline-none" href="#" data-action="analysis#fill" data-analysis-insert-param="`)
-			buf.WriteString(html.EscapeString(c.Insert))
-			buf.WriteString(`">`)
+			buf.WriteString(`<li class="p-2 hover:bg-white/25 has-[:focus]:bg-white/50 has-[:active]:bg-white/50"><a class="outline-none" href="#" data-action="analysis#fill"`)
+			fmt.Fprintf(buf, ` data-analysis-text-param="%s"`, html.EscapeString(c.Text))
+			fmt.Fprintf(buf, ` data-analysis-start-param="%d"`, c.Span.Start)
+			fmt.Fprintf(buf, ` data-analysis-end-param="%d">`, c.Span.End)
 			buf.WriteString(html.EscapeString(c.Label))
 			buf.WriteString("</a></li>\n")
 		}

From ccece1c67ca9326e8c5ab7f28352813daf0e400f Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 11 Jun 2024 13:33:54 -0700
Subject: [PATCH 11/16] Support autocomplete on take

---
 autocomplete.go      | 17 +++++++++++++++--
 autocomplete_test.go | 24 ++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/autocomplete.go b/autocomplete.go
index 6912e59..2eeada9 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -125,13 +125,21 @@ func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.Tabul
 			return nil
 		}
 		return completeColumnNames(source, prefix, columns)
-	case *parser.TopOperator:
+	case *parser.TakeOperator:
 		if prefix.End <= op.Keyword.Start {
 			return completeOperators(source, prefix, false)
 		}
-		if !op.By.IsValid() || prefix.End <= op.By.End {
+		if prefix.End <= op.Keyword.End {
 			return nil
 		}
+		return completeScope(source, prefix, letNames)
+	case *parser.TopOperator:
+		if prefix.End <= op.Keyword.Start {
+			return completeOperators(source, prefix, false)
+		}
+		if !op.By.IsValid() || prefix.End <= op.By.Start {
+			return completeScope(source, prefix, letNames)
+		}
 		return completeColumnNames(source, prefix, columns)
 	case *parser.ProjectOperator:
 		if prefix.End <= op.Keyword.Start {
@@ -171,6 +179,11 @@ func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.Tabul
 			return completeColumnNames(source, prefix, columns)
 		}
 		return nil
+	case *parser.AsOperator:
+		if prefix.End <= op.Keyword.Start {
+			return completeOperators(source, prefix, false)
+		}
+		return nil
 	default:
 		return nil
 	}
diff --git a/autocomplete_test.go b/autocomplete_test.go
index 95f1a48..1a87644 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -696,6 +696,30 @@ func TestSuggestCompletions(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "LetTake",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"People": {
+						Columns: []*AnalysisColumn{
+							{Name: "FirstName"},
+							{Name: "LastName"},
+						},
+					},
+				},
+			},
+			sourceBefore: "let foo = 5;\nPeople\n| take ",
+			want: []*Completion{
+				{
+					Label: "foo",
+					Text:  "foo",
+					Span: parser.Span{
+						Start: 27,
+						End:   27,
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {

From c4e64ae764a0d0b98c50ca741f61d7ab9e9d0887 Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 11 Jun 2024 13:39:32 -0700
Subject: [PATCH 12/16] Present let names in expression suggestions

---
 autocomplete.go      | 18 +++++++++++-------
 autocomplete_test.go | 40 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 7 deletions(-)

diff --git a/autocomplete.go b/autocomplete.go
index 2eeada9..8a5f76b 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -116,7 +116,7 @@ func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.Tabul
 		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		return completeColumnNames(source, prefix, columns)
+		return completeExpression(source, prefix, letNames, columns)
 	case *parser.SortOperator:
 		if prefix.End <= op.Keyword.Start {
 			return completeOperators(source, prefix, false)
@@ -124,7 +124,7 @@ func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.Tabul
 		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		return completeColumnNames(source, prefix, columns)
+		return completeExpression(source, prefix, letNames, columns)
 	case *parser.TakeOperator:
 		if prefix.End <= op.Keyword.Start {
 			return completeOperators(source, prefix, false)
@@ -140,7 +140,7 @@ func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.Tabul
 		if !op.By.IsValid() || prefix.End <= op.By.Start {
 			return completeScope(source, prefix, letNames)
 		}
-		return completeColumnNames(source, prefix, columns)
+		return completeExpression(source, prefix, letNames, columns)
 	case *parser.ProjectOperator:
 		if prefix.End <= op.Keyword.Start {
 			return completeOperators(source, prefix, false)
@@ -148,7 +148,7 @@ func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.Tabul
 		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		return completeColumnNames(source, prefix, columns)
+		return completeExpression(source, prefix, letNames, columns)
 	case *parser.ExtendOperator:
 		if prefix.End <= op.Keyword.Start {
 			return completeOperators(source, prefix, false)
@@ -156,7 +156,7 @@ func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.Tabul
 		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		return completeColumnNames(source, prefix, columns)
+		return completeExpression(source, prefix, letNames, columns)
 	case *parser.SummarizeOperator:
 		if prefix.End <= op.Keyword.Start {
 			return completeOperators(source, prefix, false)
@@ -164,7 +164,7 @@ func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.Tabul
 		if prefix.End <= op.Keyword.End {
 			return nil
 		}
-		return completeColumnNames(source, prefix, columns)
+		return completeExpression(source, prefix, letNames, columns)
 	case *parser.JoinOperator:
 		if prefix.End <= op.Keyword.Start {
 			return completeOperators(source, prefix, false)
@@ -176,7 +176,7 @@ func (ctx *AnalysisContext) suggestTabularExpr(source string, expr *parser.Tabul
 			return ctx.suggestTabularExpr(source, op.Right, letNames, prefix)
 		}
 		if op.On.IsValid() && prefix.End > op.On.End {
-			return completeColumnNames(source, prefix, columns)
+			return completeExpression(source, prefix, letNames, columns)
 		}
 		return nil
 	case *parser.AsOperator:
@@ -249,6 +249,10 @@ func (ctx *AnalysisContext) determineColumnsInScope(source parser.TabularDataSou
 	return columns
 }
 
+func completeExpression(source string, prefixSpan parser.Span, scope map[string]struct{}, columns []*AnalysisColumn) []*Completion {
+	return append(completeColumnNames(source, prefixSpan, columns), completeScope(source, prefixSpan, scope)...)
+}
+
 func completeColumnNames(source string, prefixSpan parser.Span, columns []*AnalysisColumn) []*Completion {
 	prefix := source[prefixSpan.Start:prefixSpan.End]
 
diff --git a/autocomplete_test.go b/autocomplete_test.go
index 1a87644..b605600 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -720,6 +720,46 @@ func TestSuggestCompletions(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "LetWhere",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"People": {
+						Columns: []*AnalysisColumn{
+							{Name: "FirstName"},
+							{Name: "LastName"},
+						},
+					},
+				},
+			},
+			sourceBefore: "let foo = \"Jane\";\nPeople\n| where FirstName = ",
+			want: []*Completion{
+				{
+					Label: "foo",
+					Text:  "foo",
+					Span: parser.Span{
+						Start: 45,
+						End:   45,
+					},
+				},
+				{
+					Label: "FirstName",
+					Text:  "FirstName",
+					Span: parser.Span{
+						Start: 45,
+						End:   45,
+					},
+				},
+				{
+					Label: "LastName",
+					Text:  "LastName",
+					Span: parser.Span{
+						Start: 45,
+						End:   45,
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {

From 046ac7cfb35616b0b8447b7e84cb06bd93ff619d Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 11 Jun 2024 13:59:44 -0700
Subject: [PATCH 13/16] Fix completion after a let statement

---
 autocomplete.go      |  9 +++++++++
 autocomplete_test.go | 24 ++++++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/autocomplete.go b/autocomplete.go
index 8a5f76b..4938d36 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -61,6 +61,15 @@ func (ctx *AnalysisContext) SuggestCompletions(source string, cursor parser.Span
 			letNames[stmt.Name.Name] = struct{}{}
 		}
 	}
+
+	// Try to figure out whether we're between a semicolon and another statement.
+	nextTokenIndex, _ := slices.BinarySearchFunc(tokens, stmts[i].Span().End, func(tok parser.Token, i int) int {
+		return cmp.Compare(tok.Span.Start, i)
+	})
+	if nextTokenIndex < len(tokens) && tokens[nextTokenIndex].Kind == parser.TokenSemi && pos >= tokens[nextTokenIndex].Span.End {
+		return ctx.completeTableNames(source, prefix)
+	}
+
 	switch stmt := stmts[i].(type) {
 	case *parser.LetStatement:
 		return ctx.suggestLetStatement(source, stmt, letNames, prefix)
diff --git a/autocomplete_test.go b/autocomplete_test.go
index b605600..47bc4c2 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -760,6 +760,30 @@ func TestSuggestCompletions(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "AfterLet",
+			context: &AnalysisContext{
+				Tables: map[string]*AnalysisTable{
+					"People": {
+						Columns: []*AnalysisColumn{
+							{Name: "FirstName"},
+							{Name: "LastName"},
+						},
+					},
+				},
+			},
+			sourceBefore: "let foo = 42;\n",
+			want: []*Completion{
+				{
+					Label: "People",
+					Text:  "People",
+					Span: parser.Span{
+						Start: 14,
+						End:   14,
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {

From d9970f1b99f28eddff19bf8d6c09715159dd8592 Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Tue, 11 Jun 2024 14:02:07 -0700
Subject: [PATCH 14/16] Benchmark completion

---
 autocomplete_test.go | 1273 +++++++++++++++++++++---------------------
 1 file changed, 643 insertions(+), 630 deletions(-)

diff --git a/autocomplete_test.go b/autocomplete_test.go
index 47bc4c2..a4c11d7 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -11,782 +11,782 @@ import (
 	"github.com/runreveal/pql/parser"
 )
 
-func TestSuggestCompletions(t *testing.T) {
-	tests := []struct {
-		name string
+var completionTests = []struct {
+	name string
 
-		context      *AnalysisContext
-		sourceBefore string
-		sourceAfter  string
+	context      *AnalysisContext
+	sourceBefore string
+	sourceAfter  string
 
-		want []*Completion
-	}{
-		{
-			name: "Empty",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "n"},
-						},
+	want []*Completion
+}{
+	{
+		name: "Empty",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "n"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "",
-			sourceAfter:  "",
-			want: []*Completion{
-				{
-					Label: "foo",
-					Text:  "foo",
-					Span: parser.Span{
-						Start: 0,
-						End:   0,
-					},
+		},
+		sourceBefore: "",
+		sourceAfter:  "",
+		want: []*Completion{
+			{
+				Label: "foo",
+				Text:  "foo",
+				Span: parser.Span{
+					Start: 0,
+					End:   0,
 				},
-				{
-					Label: "bar",
-					Text:  "bar",
-					Span: parser.Span{
-						Start: 0,
-						End:   0,
-					},
+			},
+			{
+				Label: "bar",
+				Text:  "bar",
+				Span: parser.Span{
+					Start: 0,
+					End:   0,
 				},
 			},
 		},
-		{
-			name: "InitialSourceRef",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "n"},
-						},
+	},
+	{
+		name: "InitialSourceRef",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "n"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "f",
-			sourceAfter:  "",
-			want: []*Completion{
-				{
-					Label: "foo",
-					Text:  "foo",
-					Span: parser.Span{
-						Start: 0,
-						End:   1,
-					},
+		},
+		sourceBefore: "f",
+		sourceAfter:  "",
+		want: []*Completion{
+			{
+				Label: "foo",
+				Text:  "foo",
+				Span: parser.Span{
+					Start: 0,
+					End:   1,
 				},
 			},
 		},
-		{
-			name: "SourceRefWithPipe",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "n"},
-						},
+	},
+	{
+		name: "SourceRefWithPipe",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "n"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "",
-			sourceAfter:  " | count",
-			want: []*Completion{
-				{
-					Label: "foo",
-					Text:  "foo",
-					Span: parser.Span{
-						Start: 0,
-						End:   0,
-					},
+		},
+		sourceBefore: "",
+		sourceAfter:  " | count",
+		want: []*Completion{
+			{
+				Label: "foo",
+				Text:  "foo",
+				Span: parser.Span{
+					Start: 0,
+					End:   0,
 				},
-				{
-					Label: "bar",
-					Text:  "bar",
-					Span: parser.Span{
-						Start: 0,
-						End:   0,
-					},
+			},
+			{
+				Label: "bar",
+				Text:  "bar",
+				Span: parser.Span{
+					Start: 0,
+					End:   0,
 				},
 			},
 		},
-		{
-			name: "BeforeCompleteExpr",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "n"},
-						},
+	},
+	{
+		name: "BeforeCompleteExpr",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "n"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "",
-			sourceAfter:  "o | count",
-			want: []*Completion{
-				{
-					Label: "foo",
-					Text:  "foo",
-					Span: parser.Span{
-						Start: 0,
-						End:   0,
-					},
+		},
+		sourceBefore: "",
+		sourceAfter:  "o | count",
+		want: []*Completion{
+			{
+				Label: "foo",
+				Text:  "foo",
+				Span: parser.Span{
+					Start: 0,
+					End:   0,
 				},
-				{
-					Label: "bar",
-					Text:  "bar",
-					Span: parser.Span{
-						Start: 0,
-						End:   0,
-					},
+			},
+			{
+				Label: "bar",
+				Text:  "bar",
+				Span: parser.Span{
+					Start: 0,
+					End:   0,
 				},
 			},
 		},
-		{
-			name: "BeforeSpaceThenCompleteExpr",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "n"},
-						},
+	},
+	{
+		name: "BeforeSpaceThenCompleteExpr",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "n"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "",
-			sourceAfter:  " x | count",
-			want: []*Completion{
-				{
-					Label: "foo",
-					Text:  "foo",
-					Span: parser.Span{
-						Start: 0,
-						End:   0,
-					}},
-				{
-					Label: "bar",
-					Text:  "bar",
-					Span: parser.Span{
-						Start: 0,
-						End:   0,
-					}},
-			},
-		},
-		{
-			name: "FirstOperator",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "n"},
-						},
+		},
+		sourceBefore: "",
+		sourceAfter:  " x | count",
+		want: []*Completion{
+			{
+				Label: "foo",
+				Text:  "foo",
+				Span: parser.Span{
+					Start: 0,
+					End:   0,
+				}},
+			{
+				Label: "bar",
+				Text:  "bar",
+				Span: parser.Span{
+					Start: 0,
+					End:   0,
+				}},
+		},
+	},
+	{
+		name: "FirstOperator",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "n"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "foo ",
-			sourceAfter:  "",
-			want: []*Completion{
-				{
-					Label: "as",
-					Text:  "| as",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+		},
+		sourceBefore: "foo ",
+		sourceAfter:  "",
+		want: []*Completion{
+			{
+				Label: "as",
+				Text:  "| as",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "count",
-					Text:  "| count",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "count",
+				Text:  "| count",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "extend",
-					Text:  "| extend",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "extend",
+				Text:  "| extend",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "join",
-					Text:  "| join",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "join",
+				Text:  "| join",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "limit",
-					Text:  "| limit",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "limit",
+				Text:  "| limit",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "order",
-					Text:  "| order by",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "order",
+				Text:  "| order by",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "project",
-					Text:  "| project",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "project",
+				Text:  "| project",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "sort",
-					Text:  "| sort by",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "sort",
+				Text:  "| sort by",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "summarize",
-					Text:  "| summarize",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "summarize",
+				Text:  "| summarize",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "take",
-					Text:  "| take",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "take",
+				Text:  "| take",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "top",
-					Text:  "| top",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "top",
+				Text:  "| top",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
-				{
-					Label: "where",
-					Text:  "| where",
-					Span: parser.Span{
-						Start: 4,
-						End:   4,
-					},
+			},
+			{
+				Label: "where",
+				Text:  "| where",
+				Span: parser.Span{
+					Start: 4,
+					End:   4,
 				},
 			},
 		},
-		{
-			name: "FirstOperatorAfterPipe",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "n"},
-						},
+	},
+	{
+		name: "FirstOperatorAfterPipe",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "n"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "foo |",
-			sourceAfter:  "",
-			want: []*Completion{
-				{
-					Label: "as",
-					Text:  " as",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+		},
+		sourceBefore: "foo |",
+		sourceAfter:  "",
+		want: []*Completion{
+			{
+				Label: "as",
+				Text:  " as",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "count",
-					Text:  " count",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "count",
+				Text:  " count",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "extend",
-					Text:  " extend",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "extend",
+				Text:  " extend",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "join",
-					Text:  " join",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "join",
+				Text:  " join",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "limit",
-					Text:  " limit",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "limit",
+				Text:  " limit",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "order",
-					Text:  " order by",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "order",
+				Text:  " order by",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "project",
-					Text:  " project",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "project",
+				Text:  " project",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "sort",
-					Text:  " sort by",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "sort",
+				Text:  " sort by",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "summarize",
-					Text:  " summarize",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "summarize",
+				Text:  " summarize",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "take",
-					Text:  " take",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "take",
+				Text:  " take",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "top",
-					Text:  " top",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "top",
+				Text:  " top",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
-				{
-					Label: "where",
-					Text:  " where",
-					Span: parser.Span{
-						Start: 5,
-						End:   5,
-					},
+			},
+			{
+				Label: "where",
+				Text:  " where",
+				Span: parser.Span{
+					Start: 5,
+					End:   5,
 				},
 			},
 		},
-		{
-			name: "FirstOperatorAfterPipeSpace",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "n"},
-						},
+	},
+	{
+		name: "FirstOperatorAfterPipeSpace",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "n"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "foo |  ",
-			sourceAfter:  "",
-			want: []*Completion{
-				{
-					Label: "as",
-					Text:  "as",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+		},
+		sourceBefore: "foo |  ",
+		sourceAfter:  "",
+		want: []*Completion{
+			{
+				Label: "as",
+				Text:  "as",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "count",
-					Text:  "count",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "count",
+				Text:  "count",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "extend",
-					Text:  "extend",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "extend",
+				Text:  "extend",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "join",
-					Text:  "join",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "join",
+				Text:  "join",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "limit",
-					Text:  "limit",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "limit",
+				Text:  "limit",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "order",
-					Text:  "order by",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "order",
+				Text:  "order by",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "project",
-					Text:  "project",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "project",
+				Text:  "project",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "sort",
-					Text:  "sort by",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "sort",
+				Text:  "sort by",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "summarize",
-					Text:  "summarize",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "summarize",
+				Text:  "summarize",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "take",
-					Text:  "take",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "take",
+				Text:  "take",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "top",
-					Text:  "top",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "top",
+				Text:  "top",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
-				{
-					Label: "where",
-					Text:  "where",
-					Span: parser.Span{
-						Start: 7,
-						End:   7,
-					},
+			},
+			{
+				Label: "where",
+				Text:  "where",
+				Span: parser.Span{
+					Start: 7,
+					End:   7,
 				},
 			},
 		},
-		{
-			name: "FirstOperatorPartial",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "n"},
-						},
+	},
+	{
+		name: "FirstOperatorPartial",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "n"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "foo | whe",
-			want: []*Completion{
-				{
-					Label: "where",
-					Text:  "where",
-					Span: parser.Span{
-						Start: 6,
-						End:   9,
-					},
+		},
+		sourceBefore: "foo | whe",
+		want: []*Completion{
+			{
+				Label: "where",
+				Text:  "where",
+				Span: parser.Span{
+					Start: 6,
+					End:   9,
 				},
 			},
 		},
-		{
-			name: "WhereExpression",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "name"},
-						},
+	},
+	{
+		name: "WhereExpression",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "name"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "foo | where n",
-			want: []*Completion{
-				{
-					Label: "name",
-					Text:  "name",
-					Span: parser.Span{
-						Start: 12,
-						End:   13,
-					},
+		},
+		sourceBefore: "foo | where n",
+		want: []*Completion{
+			{
+				Label: "name",
+				Text:  "name",
+				Span: parser.Span{
+					Start: 12,
+					End:   13,
 				},
 			},
 		},
-		{
-			name: "JoinExpression",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "name"},
-						},
+	},
+	{
+		name: "JoinExpression",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "name"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "foo | join (b",
-			want: []*Completion{
-				{
-					Label: "bar",
-					Text:  "bar",
-					Span: parser.Span{
-						Start: 12,
-						End:   13,
-					},
+		},
+		sourceBefore: "foo | join (b",
+		want: []*Completion{
+			{
+				Label: "bar",
+				Text:  "bar",
+				Span: parser.Span{
+					Start: 12,
+					End:   13,
 				},
 			},
 		},
-		{
-			name: "JoinExpressionOn",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"foo": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-							{Name: "name"},
-						},
+	},
+	{
+		name: "JoinExpressionOn",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"foo": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
+						{Name: "name"},
 					},
-					"bar": {
-						Columns: []*AnalysisColumn{
-							{Name: "id"},
-						},
+				},
+				"bar": {
+					Columns: []*AnalysisColumn{
+						{Name: "id"},
 					},
 				},
 			},
-			sourceBefore: "foo | join (bar) on i",
-			want: []*Completion{
-				{
-					Label: "id",
-					Text:  "id",
-					Span: parser.Span{
-						Start: 20,
-						End:   21,
-					},
+		},
+		sourceBefore: "foo | join (bar) on i",
+		want: []*Completion{
+			{
+				Label: "id",
+				Text:  "id",
+				Span: parser.Span{
+					Start: 20,
+					End:   21,
 				},
 			},
 		},
-		{
-			name: "Project",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"People": {
-						Columns: []*AnalysisColumn{
-							{Name: "FirstName"},
-							{Name: "LastName"},
-						},
+	},
+	{
+		name: "Project",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"People": {
+					Columns: []*AnalysisColumn{
+						{Name: "FirstName"},
+						{Name: "LastName"},
 					},
 				},
 			},
-			sourceBefore: "People\n| project F",
-			sourceAfter:  ", LastName",
-			want: []*Completion{
-				{
-					Label: "FirstName",
-					Text:  "FirstName",
-					Span: parser.Span{
-						Start: 17,
-						End:   18,
-					},
+		},
+		sourceBefore: "People\n| project F",
+		sourceAfter:  ", LastName",
+		want: []*Completion{
+			{
+				Label: "FirstName",
+				Text:  "FirstName",
+				Span: parser.Span{
+					Start: 17,
+					End:   18,
 				},
 			},
 		},
-		{
-			name: "LetTake",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"People": {
-						Columns: []*AnalysisColumn{
-							{Name: "FirstName"},
-							{Name: "LastName"},
-						},
+	},
+	{
+		name: "LetTake",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"People": {
+					Columns: []*AnalysisColumn{
+						{Name: "FirstName"},
+						{Name: "LastName"},
 					},
 				},
 			},
-			sourceBefore: "let foo = 5;\nPeople\n| take ",
-			want: []*Completion{
-				{
-					Label: "foo",
-					Text:  "foo",
-					Span: parser.Span{
-						Start: 27,
-						End:   27,
-					},
+		},
+		sourceBefore: "let foo = 5;\nPeople\n| take ",
+		want: []*Completion{
+			{
+				Label: "foo",
+				Text:  "foo",
+				Span: parser.Span{
+					Start: 27,
+					End:   27,
 				},
 			},
 		},
-		{
-			name: "LetWhere",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"People": {
-						Columns: []*AnalysisColumn{
-							{Name: "FirstName"},
-							{Name: "LastName"},
-						},
+	},
+	{
+		name: "LetWhere",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"People": {
+					Columns: []*AnalysisColumn{
+						{Name: "FirstName"},
+						{Name: "LastName"},
 					},
 				},
 			},
-			sourceBefore: "let foo = \"Jane\";\nPeople\n| where FirstName = ",
-			want: []*Completion{
-				{
-					Label: "foo",
-					Text:  "foo",
-					Span: parser.Span{
-						Start: 45,
-						End:   45,
-					},
+		},
+		sourceBefore: "let foo = \"Jane\";\nPeople\n| where FirstName = ",
+		want: []*Completion{
+			{
+				Label: "foo",
+				Text:  "foo",
+				Span: parser.Span{
+					Start: 45,
+					End:   45,
 				},
-				{
-					Label: "FirstName",
-					Text:  "FirstName",
-					Span: parser.Span{
-						Start: 45,
-						End:   45,
-					},
+			},
+			{
+				Label: "FirstName",
+				Text:  "FirstName",
+				Span: parser.Span{
+					Start: 45,
+					End:   45,
 				},
-				{
-					Label: "LastName",
-					Text:  "LastName",
-					Span: parser.Span{
-						Start: 45,
-						End:   45,
-					},
+			},
+			{
+				Label: "LastName",
+				Text:  "LastName",
+				Span: parser.Span{
+					Start: 45,
+					End:   45,
 				},
 			},
 		},
-		{
-			name: "AfterLet",
-			context: &AnalysisContext{
-				Tables: map[string]*AnalysisTable{
-					"People": {
-						Columns: []*AnalysisColumn{
-							{Name: "FirstName"},
-							{Name: "LastName"},
-						},
+	},
+	{
+		name: "AfterLet",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"People": {
+					Columns: []*AnalysisColumn{
+						{Name: "FirstName"},
+						{Name: "LastName"},
 					},
 				},
 			},
-			sourceBefore: "let foo = 42;\n",
-			want: []*Completion{
-				{
-					Label: "People",
-					Text:  "People",
-					Span: parser.Span{
-						Start: 14,
-						End:   14,
-					},
+		},
+		sourceBefore: "let foo = 42;\n",
+		want: []*Completion{
+			{
+				Label: "People",
+				Text:  "People",
+				Span: parser.Span{
+					Start: 14,
+					End:   14,
 				},
 			},
 		},
-	}
+	},
+}
 
-	for _, test := range tests {
+func TestSuggestCompletions(t *testing.T) {
+	for _, test := range completionTests {
 		t.Run(test.name, func(t *testing.T) {
 			got := test.context.SuggestCompletions(test.sourceBefore+test.sourceAfter, parser.Span{
 				Start: len(test.sourceBefore),
@@ -810,3 +810,16 @@ func TestSuggestCompletions(t *testing.T) {
 		})
 	}
 }
+
+func BenchmarkSuggestCompletions(b *testing.B) {
+	for _, test := range completionTests {
+		b.Run(test.name, func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				test.context.SuggestCompletions(test.sourceBefore+test.sourceAfter, parser.Span{
+					Start: len(test.sourceBefore),
+					End:   len(test.sourceBefore),
+				})
+			}
+		})
+	}
+}

From 7d5e88e0d4d9346dd1a37e4e0f46bc392cee92be Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Wed, 12 Jun 2024 10:14:52 -0700
Subject: [PATCH 15/16] Remove testing GUI

---
 cmd/pql-playground/app.js     |  67 ------------------
 cmd/pql-playground/index.html |  56 ---------------
 cmd/pql-playground/main.go    | 128 ----------------------------------
 3 files changed, 251 deletions(-)
 delete mode 100644 cmd/pql-playground/app.js
 delete mode 100644 cmd/pql-playground/index.html
 delete mode 100644 cmd/pql-playground/main.go

diff --git a/cmd/pql-playground/app.js b/cmd/pql-playground/app.js
deleted file mode 100644
index 01d8105..0000000
--- a/cmd/pql-playground/app.js
+++ /dev/null
@@ -1,67 +0,0 @@
-import { Application, Controller } from 'https://unpkg.com/@hotwired/stimulus/dist/stimulus.js';
-
-window.Stimulus = Application.start();
-Stimulus.register('analysis', class extends Controller {
-  static targets = ['list', 'editor', 'output'];
-  static values = {
-    compileHref: String,
-    suggestHref: String,
-  };
-
-  /**
-   * @param {Event} event
-   * @return {Promise<void>}
-   */
-  async compile(event) {
-    const formData = new URLSearchParams();
-    formData.append('source', this.editorTarget.value);
-    const response = await fetch(this.compileHrefValue, {
-      method: 'POST',
-      body: formData,
-    });
-    if (!response.ok) {
-      return;
-    }
-    this.outputTarget.innerHTML = await response.text();
-    this.outputTarget.hidden = false;
-  }
-
-  /**
-   * @param {Event} event
-   * @return {Promise<void>}
-   */
-  async suggest(event) {
-    const formData = new URLSearchParams();
-    formData.append('source', this.editorTarget.value);
-    formData.append('start', this.editorTarget.selectionStart);
-    formData.append('end', this.editorTarget.selectionEnd);
-    const response = await fetch(this.suggestHrefValue, {
-      method: 'POST',
-      body: formData,
-    });
-    if (!response.ok) {
-      return;
-    }
-    this.listTarget.innerHTML = await response.text();
-    this.listTarget.hidden = false;
-  }
-
-  clear() {
-    this.listTarget.innerHTML = '';
-    this.listTarget.hidden = true;
-    this.editorTarget.focus();
-  }
-
-  /**
-   * @param {Event} event
-   */
-  fill(event) {
-    this.editorTarget.value = this.editorTarget.value.substring(0, event.params.start) +
-      event.params.text +
-      this.editorTarget.value.substring(event.params.end);
-    const i = event.params.start + event.params.insert.length;
-    this.editorTarget.setSelectionRange(i, i);
-
-    this.clear();
-  }
-});
diff --git a/cmd/pql-playground/index.html b/cmd/pql-playground/index.html
deleted file mode 100644
index c119617..0000000
--- a/cmd/pql-playground/index.html
+++ /dev/null
@@ -1,56 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <title>pql Autocomplete Playground</title>
-    <script
-      type="module"
-      src="https://unpkg.com/@hotwired/stimulus/dist/stimulus.js"
-    ></script>
-    <script type="module" src="/app.js"></script>
-    <script src="https://cdn.tailwindcss.com"></script>
-  </head>
-  <body class="bg-slate-200">
-    <div
-      class="p-16"
-      data-controller="analysis"
-      data-analysis-compile-href-value="/compile"
-      data-analysis-suggest-href-value="/suggest"
-    >
-      <form
-        data-action="analysis#compile:prevent keydown.ctrl+enter->analysis#compile"
-        class="mx-auto my-8 grid grid-cols-[2fr_1fr] max-w-[90ch] gap-x-4 gap-y-8"
-      >
-        <div>
-          <textarea
-            name="source"
-            rows="20"
-            class="p-2 border border-slate-400 font-mono w-full bg-slate-50"
-            autofocus
-            data-analysis-target="editor"
-            data-action="input->analysis#clear selectionchange->analysis#clear keydown.ctrl+space->analysis#suggest keydown.esc->analysis#clear"
-          ></textarea>
-        </div>
-
-        <div>
-          <ul
-            class="bg-slate-700 text-slate-100"
-            data-analysis-target="list"
-            data-action="keydown.esc->analysis#clear"
-            hidden
-          ></ul>
-        </div>
-
-        <div class="text-center">
-          <input
-            type="submit"
-            value="Compile"
-            class="inline-block px-8 py-4 text-lg font-bold bg-blue-600 text-white rounded-md outline-none focus:ring focus:ring-orange-400"
-          />
-        </div>
-      </form>
-
-      <div class="mx-auto my-16 border-t border-slate-400 p-8 max-w-[80ch]" data-analysis-target="output" hidden></div>
-    </div>
-  </body>
-</html>
diff --git a/cmd/pql-playground/main.go b/cmd/pql-playground/main.go
deleted file mode 100644
index 6beb574..0000000
--- a/cmd/pql-playground/main.go
+++ /dev/null
@@ -1,128 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"embed"
-	"errors"
-	"fmt"
-	"html"
-	"io"
-	"io/fs"
-	"log"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/runreveal/pql"
-	"github.com/runreveal/pql/parser"
-)
-
-//go:embed index.html
-//go:embed app.js
-var static embed.FS
-
-func main() {
-	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		serveFileFS(w, r, static, "index.html")
-	})
-	http.HandleFunc("/app.js", func(w http.ResponseWriter, r *http.Request) {
-		serveFileFS(w, r, static, "app.js")
-	})
-	http.HandleFunc("/compile", compile)
-	http.HandleFunc("/suggest", suggest)
-
-	log.Println("Listening")
-	http.ListenAndServe(":8080", nil)
-}
-
-func compile(w http.ResponseWriter, r *http.Request) {
-	sql, err := pql.Compile(r.FormValue("source"))
-	buf := new(bytes.Buffer)
-	if err != nil {
-		buf.WriteString(`<div class="italic text-red-900">`)
-		buf.WriteString(html.EscapeString(err.Error()))
-		buf.WriteString("</div>")
-	} else {
-		buf.WriteString(`<div class="font-mono bg-slate-300 text-black p-4">`)
-		for _, line := range strings.Split(sql, "\n") {
-			buf.WriteString(`<pre class="whitespace-pre-line ps-8 -indent-8"><code>`)
-			buf.WriteString(html.EscapeString(line))
-			buf.WriteString("</code></pre>\n")
-		}
-		buf.WriteString("</div>\n")
-	}
-	writeBuffer(w, buf)
-}
-
-func suggest(w http.ResponseWriter, r *http.Request) {
-	ctx := &pql.AnalysisContext{
-		Tables: map[string]*pql.AnalysisTable{
-			"People": {
-				Columns: []*pql.AnalysisColumn{
-					{Name: "FirstName"},
-					{Name: "LastName"},
-					{Name: "PhoneNumber"},
-				},
-			},
-		},
-	}
-	start, err := strconv.Atoi(r.FormValue("start"))
-	if err != nil {
-		http.Error(w, "start: "+err.Error(), http.StatusUnprocessableEntity)
-		return
-	}
-	end, err := strconv.Atoi(r.FormValue("end"))
-	if err != nil {
-		http.Error(w, "end: "+err.Error(), http.StatusUnprocessableEntity)
-		return
-	}
-	completions := ctx.SuggestCompletions(r.FormValue("source"), parser.Span{
-		Start: start,
-		End:   end,
-	})
-
-	buf := new(bytes.Buffer)
-	if len(completions) == 0 {
-		buf.WriteString(`<li class="p-2">No completions.</li>`)
-	} else {
-		for _, c := range completions {
-			buf.WriteString(`<li class="p-2 hover:bg-white/25 has-[:focus]:bg-white/50 has-[:active]:bg-white/50"><a class="outline-none" href="#" data-action="analysis#fill"`)
-			fmt.Fprintf(buf, ` data-analysis-text-param="%s"`, html.EscapeString(c.Text))
-			fmt.Fprintf(buf, ` data-analysis-start-param="%d"`, c.Span.Start)
-			fmt.Fprintf(buf, ` data-analysis-end-param="%d">`, c.Span.End)
-			buf.WriteString(html.EscapeString(c.Label))
-			buf.WriteString("</a></li>\n")
-		}
-	}
-	writeBuffer(w, buf)
-}
-
-func writeBuffer(w http.ResponseWriter, buf *bytes.Buffer) {
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
-	w.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
-	io.Copy(w, buf)
-}
-
-func serveFileFS(w http.ResponseWriter, r *http.Request, fsys fs.FS, name string) {
-	f, err := fsys.Open(name)
-	if errors.Is(err, fs.ErrNotExist) {
-		http.NotFound(w, r)
-		return
-	}
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	defer f.Close()
-	content, ok := f.(io.ReadSeeker)
-	if !ok {
-		contentBytes, err := io.ReadAll(f)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-		content = bytes.NewReader(contentBytes)
-	}
-	http.ServeContent(w, r, name, time.Time{}, content)
-}

From eb93325a958ac4b28b6eadad38ac416125162ebb Mon Sep 17 00:00:00 2001
From: Ross Light <ross@256lights.llc>
Date: Fri, 14 Jun 2024 20:33:01 -0700
Subject: [PATCH 16/16] Make filtering case-insensitive

---
 autocomplete.go      | 32 +++++++++++++++++++++++++----
 autocomplete_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+), 4 deletions(-)

diff --git a/autocomplete.go b/autocomplete.go
index 4938d36..0505607 100644
--- a/autocomplete.go
+++ b/autocomplete.go
@@ -7,6 +7,7 @@ import (
 	"cmp"
 	"slices"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/runreveal/pql/parser"
 )
@@ -267,7 +268,7 @@ func completeColumnNames(source string, prefixSpan parser.Span, columns []*Analy
 
 	result := make([]*Completion, 0, len(columns))
 	for _, col := range columns {
-		if strings.HasPrefix(col.Name, prefix) {
+		if hasFoldPrefix(col.Name, prefix) {
 			result = append(result, &Completion{
 				Label: col.Name,
 				Text:  col.Name,
@@ -283,7 +284,7 @@ func completeScope(source string, prefixSpan parser.Span, scope map[string]struc
 
 	result := make([]*Completion, 0, len(scope))
 	for name := range scope {
-		if strings.HasPrefix(name, prefix) {
+		if hasFoldPrefix(name, prefix) {
 			result = append(result, &Completion{
 				Label: name,
 				Text:  name,
@@ -299,7 +300,7 @@ func (ctx *AnalysisContext) completeTableNames(source string, prefixSpan parser.
 
 	result := make([]*Completion, 0, len(ctx.Tables))
 	for tableName := range ctx.Tables {
-		if strings.HasPrefix(tableName, prefix) {
+		if hasFoldPrefix(tableName, prefix) {
 			result = append(result, &Completion{
 				Label: tableName,
 				Text:  tableName,
@@ -332,7 +333,7 @@ func completeOperators(source string, prefixSpan parser.Span, includePipe bool)
 
 	result := make([]*Completion, 0, len(sortedOperatorNames))
 	for _, name := range sortedOperatorNames {
-		if !strings.HasPrefix(name, prefix) {
+		if !hasFoldPrefix(name, prefix) {
 			continue
 		}
 		c := &Completion{
@@ -395,3 +396,26 @@ func isCompletableToken(kind parser.TokenKind) bool {
 		kind == parser.TokenIn ||
 		kind == parser.TokenBy
 }
+
+// hasFoldPrefix reports whether s starts with the given prefix,
+// ignoring differences in case.
+func hasFoldPrefix(s, prefix string) bool {
+	n := utf8.RuneCountInString(prefix)
+
+	// Find the end of the first n runes in s.
+	// If s does not have that many runes,
+	// then it can't have the prefix.
+	if len(s) < n {
+		return false
+	}
+	var prefixLen int
+	for i := 0; i < n; i++ {
+		_, sz := utf8.DecodeRuneInString(s[prefixLen:])
+		if sz == 0 {
+			return false
+		}
+		prefixLen += sz
+	}
+
+	return strings.EqualFold(s[:prefixLen], prefix)
+}
diff --git a/autocomplete_test.go b/autocomplete_test.go
index a4c11d7..99a7456 100644
--- a/autocomplete_test.go
+++ b/autocomplete_test.go
@@ -783,6 +783,30 @@ var completionTests = []struct {
 			},
 		},
 	},
+	{
+		name: "CaseInsensitive",
+		context: &AnalysisContext{
+			Tables: map[string]*AnalysisTable{
+				"People": {
+					Columns: []*AnalysisColumn{
+						{Name: "FirstName"},
+						{Name: "LastName"},
+					},
+				},
+			},
+		},
+		sourceBefore: "People | where fI",
+		want: []*Completion{
+			{
+				Label: "FirstName",
+				Text:  "FirstName",
+				Span: parser.Span{
+					Start: 15,
+					End:   17,
+				},
+			},
+		},
+	},
 }
 
 func TestSuggestCompletions(t *testing.T) {
@@ -823,3 +847,28 @@ func BenchmarkSuggestCompletions(b *testing.B) {
 		})
 	}
 }
+
+func TestHasFoldPrefix(t *testing.T) {
+	tests := []struct {
+		s      string
+		prefix string
+		want   bool
+	}{
+		{"", "", true},
+		{"abc", "", true},
+		{"abc", "a", true},
+		{"abc", "A", true},
+		{"Abc", "a", true},
+		{"Abc", "A", true},
+		{"Abc", "x", false},
+		{"", "abc", false},
+		{"ABC", "abc", true},
+		{"abc", "ABC", true},
+		{"ABC", "abcd", false},
+	}
+	for _, test := range tests {
+		if got := hasFoldPrefix(test.s, test.prefix); got != test.want {
+			t.Errorf("hasFoldPrefix(%q, %q) = %t; want %t", test.s, test.prefix, got, test.want)
+		}
+	}
+}