TokenApplication API

Sources/Neon/Highlighter.swift

@@ -2,13 +2,6 @@ import Foundation
 import Rearrange
 import os.log
 
-public protocol TextSystemInterface {
-    func clearStyle(in range: NSRange)
-    func applyStyle(to token: Token)
-
-    var length: Int { get }
-    var visibleRange: NSRange { get }
-}
 
 public class Highlighter {
     public var textInterface: TextSystemInterface
@@ -17,14 +10,12 @@ public class Highlighter {
     private var pendingSet: IndexSet
     private var log: OSLog
     public var tokenProvider: TokenProvider
-    private var failed: Bool
 
     public init(textInterface: TextSystemInterface, tokenProvider: TokenProvider? = nil) {
         self.textInterface = textInterface
         self.validSet = IndexSet()
         self.pendingSet = IndexSet()
         self.tokenProvider = tokenProvider ?? { _, block in block(.success([]))}
-        self.failed = false
 
         self.log = OSLog(subsystem: "com.chimehq.Neon", category: "Highlighter")
     }
@@ -34,7 +25,10 @@ extension Highlighter {
     public func invalidate(_ set: IndexSet) {
         dispatchPrecondition(condition: .onQueue(.main))
 
-        self.failed = false
+        if set.isEmpty {
+            return
+        }
+
         validSet.subtract(set)
         pendingSet.subtract(set)
 
@@ -123,19 +117,19 @@ extension Highlighter {
     }
 
     private func makeNextTokenRequest() {
-        guard failed == false else { return }
-
         guard let range = nextNeededTokenRange() else { return }
 
         self.pendingSet.insert(range: range)
 
+        // this can be called 0 or more times
         tokenProvider(range) { result in
+            dispatchPrecondition(condition: .onQueue(.main))
+
             switch result {
             case .failure(let error):
                 os_log("failed to get tokens: %{public}@", log: self.log, type: .error, String(describing: error))
 
                 DispatchQueue.main.async {
-                    self.failed = true
                     self.pendingSet.remove(integersIn: range)
                 }
             case .success(let tokens):
@@ -156,22 +150,8 @@ extension Highlighter {
 
         let receivedSet = IndexSet(integersIn: range)
 
-        applyStyle(with: tokenApplication, in: receivedSet)
-    }
-
-    private func applyStyle(with tokenApplication: TokenApplication, in set: IndexSet) {
-        precondition(set.isEmpty == false)
-
-        if tokenApplication.action == .replace {
-            for range in set.nsRangeView {
-                textInterface.clearStyle(in: range)
-            }
-        }
-
-        for token in tokenApplication.tokens {
-            textInterface.applyStyle(to: token)
-        }
+        textInterface.apply(tokenApplication, to: receivedSet)
 
-        validSet.formUnion(set)
+        validSet.formUnion(receivedSet)
     }
 }

Sources/Neon/TextSystemInterface.swift

@@ -0,0 +1,31 @@
+import Foundation
+
+public protocol TextSystemInterface {
+    func clearStyle(in range: NSRange)
+    func applyStyle(to token: Token)
+
+    var length: Int { get }
+    var visibleRange: NSRange { get }
+}
+
+public extension TextSystemInterface {
+    func clearStyles(in set: IndexSet) {
+        for range in set.nsRangeView {
+            clearStyle(in: range)
+        }
+    }
+
+    func applyStyles(to tokens: [Token]) {
+        for token in tokens {
+            applyStyle(to: token)
+        }
+    }
+
+    func apply(_ tokenApplication: TokenApplication, to set: IndexSet) {
+        if tokenApplication.action == .replace {
+            clearStyles(in: set)
+        }
+
+        applyStyles(to: tokenApplication.tokens)
+    }
+}

Sources/Neon/Token.swift

@@ -16,7 +16,7 @@ extension Token: Hashable {
 public struct TokenApplication {
     public enum Action {
         case replace
-        case merge
+        case apply
     }
 
     public let tokens: [Token]
@@ -26,6 +26,8 @@ public struct TokenApplication {
         self.tokens = tokens
         self.action = action
     }
+
+    public static let noChange = TokenApplication(tokens: [], action: .apply)
 }
 
 extension TokenApplication: ExpressibleByArrayLiteral {
@@ -36,4 +38,12 @@ extension TokenApplication: ExpressibleByArrayLiteral {
     }
 }
 
+/// A source of `Token` information
+///
+/// The callback argument of this function is a little funny. It is ok
+/// to invoke it 0 to N times. Always invoke it on the main queue.
+///
+/// Note: failures are always assumed to be transient. The
+/// best way to indicate a permenant failure is to just return
+/// `TokenApplication.noChange`.
 public typealias TokenProvider = (NSRange, @escaping (Result<TokenApplication, Error>) -> Void) -> Void

Sources/Neon/TreeSitterClient.swift

@@ -260,8 +260,9 @@ extension TreeSitterClient {
     /// Determine if it is likely that a synchronous query will execute quickly
     public func canAttemptSynchronousQuery(in range: NSRange) -> Bool {
         let largeRange = range.length > synchronousLengthThreshold
+        let largeLocation = range.max > synchronousContentLengthThreshold
 
-        return hasQueuedWork == false && largeRange == false
+        return (hasQueuedWork || largeRange || largeLocation) == false
     }
 
     /// Executes a query and returns a ResolvingQueryCursor