Optimise didChangeWatchedFiles for large file changes

Right now, didChangeWatchedFiles applies one-by-one on each file. Each time it does this, it marks the file as dirty (when indexing is enabled), then attempts to parse the file.

This is non-optimal if there are a large set of related file changes. Each time, we'll mark as dirty, then parse a single file (which will parse its dependencies). Then, move to the next file, mark it as dirty and repeat. But when we mark it as dirty, we've invalidated all the work we've just done.

We fix this by deferring parsing to after all the changes have been processed, and the dirty list constructed. This prevents unnecessary invalidation.

Also, we now skip ignored files from indexing.

Author: JohnnyMorganz

CHANGELOG.md

@@ -9,6 +9,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Changed
 
 - Sync to upstream Luau 0.666
+- Optimized re-indexing on changed files when large number of changes processed at once
+- Watched files re-indexing now respects ignore globs specification
 
 ### Fixed
 

src/LanguageServer.cpp

@@ -733,11 +733,18 @@ void LanguageServer::onDidChangeWorkspaceFolders(const lsp::DidChangeWorkspaceFo
 
 void LanguageServer::onDidChangeWatchedFiles(const lsp::DidChangeWatchedFilesParams& params)
 {
+    Luau::DenseHashMap<WorkspaceFolderPtr, std::vector<lsp::FileEvent>> workspaceChanges{nullptr};
+
     for (const auto& change : params.changes)
     {
         auto workspace = findWorkspace(change.uri);
-        workspace->onDidChangeWatchedFiles(change);
+        if (!workspaceChanges.find(workspace))
+            workspaceChanges[workspace] = {};
+        workspaceChanges[workspace].push_back(change);
     }
+
+    for (const auto& [workspace, changes] : workspaceChanges)
+        workspace->onDidChangeWatchedFiles(changes);
 }
 
 Response LanguageServer::onShutdown([[maybe_unused]] const id_type& id)

src/Workspace.cpp

@@ -67,18 +67,25 @@ void WorkspaceFolder::closeTextDocument(const lsp::DocumentUri& uri)
         clearDiagnosticsForFile(uri);
 }
 
-void WorkspaceFolder::clearDiagnosticsForFile(const lsp::DocumentUri& uri)
+void WorkspaceFolder::clearDiagnosticsForFiles(const std::vector<lsp::DocumentUri>& uris) const
 {
     if (!client->capabilities.textDocument || !client->capabilities.textDocument->diagnostic)
     {
-        client->publishDiagnostics(lsp::PublishDiagnosticsParams{uri, std::nullopt, {}});
+        for (const auto& uri : uris)
+            client->publishDiagnostics(lsp::PublishDiagnosticsParams{uri, std::nullopt, {}});
     }
     else if (client->workspaceDiagnosticsToken)
     {
-        lsp::WorkspaceDocumentDiagnosticReport documentReport;
-        documentReport.uri = uri;
-        documentReport.kind = lsp::DocumentDiagnosticReportKind::Full;
-        lsp::WorkspaceDiagnosticReportPartialResult report{{documentReport}};
+        std::vector<lsp::WorkspaceDocumentDiagnosticReport> reports;
+        reports.reserve(uris.size());
+        for (const auto& uri : uris)
+        {
+            lsp::WorkspaceDocumentDiagnosticReport report;
+            report.uri = uri;
+            report.kind = lsp::DocumentDiagnosticReportKind::Full;
+            reports.push_back(report);
+        }
+        lsp::WorkspaceDiagnosticReportPartialResult report{reports};
         client->sendProgress({client->workspaceDiagnosticsToken.value(), report});
     }
     else
@@ -87,52 +94,66 @@ void WorkspaceFolder::clearDiagnosticsForFile(const lsp::DocumentUri& uri)
     }
 }
 
-void WorkspaceFolder::onDidChangeWatchedFiles(const lsp::FileEvent& change)
+void WorkspaceFolder::clearDiagnosticsForFile(const lsp::DocumentUri& uri)
+{
+    clearDiagnosticsForFiles({uri});
+}
+
+void WorkspaceFolder::onDidChangeWatchedFiles(const std::vector<lsp::FileEvent>& changes)
 {
-    auto filePath = change.uri.fsPath();
+    client->sendTrace("workspace: processing " + std::to_string(changes.size()) + " watched files changes");
+
     auto config = client->getConfiguration(rootUri);
 
-    platform->onDidChangeWatchedFiles(change);
+    std::vector<Luau::ModuleName> dirtyFiles;
+    std::vector<Uri> deletedFiles;
 
-    if (filePath.filename() == ".luaurc")
+    for (const auto& change : changes)
     {
-        client->sendLogMessage(lsp::MessageType::Info, "Acknowledge config changed for workspace " + name + ", clearing configuration cache");
-        fileResolver.clearConfigCache();
+        auto filePath = change.uri.fsPath();
 
-        // Recompute diagnostics
-        recomputeDiagnostics(config);
-    }
-    else if (filePath.extension() == ".lua" || filePath.extension() == ".luau")
-    {
-        // Notify if it was a definitions file
-        if (isDefinitionFile(filePath, config))
-        {
-            client->sendWindowMessage(
-                lsp::MessageType::Info, "Detected changes to global definitions files. Please reload your workspace for this to take effect");
-            return;
-        }
+        platform->onDidChangeWatchedFiles(change);
 
-        // Index the workspace on changes
-        // We only update the require graph. We do not perform type checking
-        if (config.index.enabled && isConfigured)
+        if (filePath.filename() == ".luaurc")
         {
-            auto moduleName = fileResolver.getModuleName(change.uri);
+            client->sendLogMessage(lsp::MessageType::Info, "Acknowledge config changed for workspace " + name + ", clearing configuration cache");
+            fileResolver.clearConfigCache();
 
-            std::vector<Luau::ModuleName> markedDirty{};
-            frontend.markDirty(moduleName, &markedDirty);
+            // Recompute diagnostics
+            recomputeDiagnostics(config);
+        }
+        else if (filePath.extension() == ".lua" || filePath.extension() == ".luau")
+        {
+            // Notify if it was a definitions file
+            if (isDefinitionFile(filePath, config))
+            {
+                client->sendWindowMessage(
+                    lsp::MessageType::Info, "Detected changes to global definitions files. Please reload your workspace for this to take effect");
+                continue;
+            }
+            else if (isIgnoredFile(filePath, config))
+            {
+                continue;
+            }
 
-            if (change.type == lsp::FileChangeType::Created)
-                frontend.parse(moduleName);
+            // Index the workspace on changes
+            if (config.index.enabled && isConfigured)
+            {
+                auto moduleName = fileResolver.getModuleName(change.uri);
+                frontend.markDirty(moduleName, &dirtyFiles);
+            }
 
-            // Re-check the reverse dependencies
-            for (const auto& reverseDep : markedDirty)
-                frontend.parse(reverseDep);
+            if (change.type == lsp::FileChangeType::Deleted)
+                deletedFiles.push_back(change.uri);
         }
-
-        // Clear the diagnostics for the file in case it was not managed
-        if (change.type == lsp::FileChangeType::Deleted)
-            clearDiagnosticsForFile(change.uri);
     }
+
+    // Parse require graph for files if indexing enabled
+    for (const auto& dirtyModule : dirtyFiles)
+        frontend.parse(dirtyModule);
+
+    // Clear the diagnostics for files in case it was not managed
+    clearDiagnosticsForFiles(deletedFiles);
 }
 
 /// When using lexically_relative, if the root-dir does not match, it will return a default-constructed path.

src/include/LSP/Workspace.hpp

@@ -60,7 +60,7 @@ class WorkspaceFolder
         const lsp::DocumentUri& uri, const lsp::DidChangeTextDocumentParams& params, std::vector<Luau::ModuleName>* markedDirty = nullptr);
     void closeTextDocument(const lsp::DocumentUri& uri);
 
-    void onDidChangeWatchedFiles(const lsp::FileEvent& change);
+    void onDidChangeWatchedFiles(const std::vector<lsp::FileEvent>& changes);
 
     /// Whether the file has been marked as ignored by any of the ignored lists in the configuration
     bool isIgnoredFile(const std::filesystem::path& path, const std::optional<ClientConfiguration>& givenConfig = std::nullopt);
@@ -74,12 +74,13 @@ class WorkspaceFolder
     void recomputeDiagnostics(const ClientConfiguration& config);
     void pushDiagnostics(const lsp::DocumentUri& uri, const size_t version);
 
+    void clearDiagnosticsForFiles(const std::vector<lsp::DocumentUri>& uri) const;
     void clearDiagnosticsForFile(const lsp::DocumentUri& uri);
 
     void indexFiles(const ClientConfiguration& config);
 
     Luau::CheckResult checkSimple(const Luau::ModuleName& moduleName);
-    void checkStrict(const Luau::ModuleName& moduleName, bool forAutocomplete = true);
+    Luau::CheckResult checkStrict(const Luau::ModuleName& moduleName, bool forAutocomplete = true);
     // TODO: Clip once new type solver is live
     const Luau::ModulePtr getModule(const Luau::ModuleName& moduleName, bool forAutocomplete = false) const;