Refactor MarkdownToolbar and CodeUtils for improved token processing

Author: dan-niles

workspaces/ballerina/ballerina-side-panel/src/components/editors/ExpandedEditor/controls/MarkdownToolbar.tsx

@@ -20,7 +20,6 @@ import React from "react";
 import styled from "@emotion/styled";
 import { ThemeColors, Icon, Switch } from "@wso2/ui-toolkit";
 import "@github/markdown-toolbar-element";
-import { HelperPaneToggleButton } from "../../../editors/MultiModeExpressionEditor/ChipExpressionEditor/components/HelperPaneToggleButton";
 
 // Type declarations for GitHub markdown toolbar custom elements
 declare global {
@@ -108,12 +107,6 @@ interface MarkdownToolbarProps {
     isPreviewMode?: boolean;
     /** Callback to toggle preview mode */
     onTogglePreview?: () => void;
-    /** Helper pane toggle button props (optional) */
-    helperPaneToggle?: {
-        ref: React.RefObject<HTMLButtonElement>;
-        isOpen: boolean;
-        onClick: () => void;
-    };
 }
 
 /**
@@ -123,8 +116,7 @@ interface MarkdownToolbarProps {
 export const MarkdownToolbar: React.FC<MarkdownToolbarProps> = ({
     textareaId,
     isPreviewMode = false,
-    onTogglePreview,
-    helperPaneToggle
+    onTogglePreview
 }) => {
     return (
         <ToolbarContainer>

workspaces/ballerina/ballerina-side-panel/src/components/editors/ExpandedEditor/utils/transformToMarkdown.ts

@@ -16,6 +16,7 @@
  * under the License.
  */
 
+import { iterateTokenStream } from "../../MultiModeExpressionEditor/ChipExpressionEditor/CodeUtils";
 import { TokenType } from "../../MultiModeExpressionEditor/ChipExpressionEditor/types";
 import { getParsedExpressionTokens, detectTokenPatterns } from "../../MultiModeExpressionEditor/ChipExpressionEditor/utils";
 
@@ -50,96 +51,63 @@ export const transformExpressionToMarkdown = (
     }
 
     try {
-        // Parse tokens to get absolute positions
         const parsedTokens = getParsedExpressionTokens(tokenStream, expression);
-
-        // Detect compound token patterns (variables and documents)
         const compounds = detectTokenPatterns(parsedTokens, expression);
-
         const renderableTokens: RenderableToken[] = [];
 
-        // Create a set of token indices that are part of compounds
-        const compoundTokenIndices = new Set<number>();
-        for (const compound of compounds) {
-            for (let i = compound.startIndex; i <= compound.endIndex; i++) {
-                compoundTokenIndices.add(i);
-            }
-        }
-
-        // Process compound tokens
-        for (const compound of compounds) {
-            let chipTag: string;
-
-            if (compound.tokenType === TokenType.DOCUMENT && compound.metadata.documentType) {
-                chipTag = createChipTag('document', compound.metadata.content, compound.metadata.documentType);
-            } else if (compound.tokenType === TokenType.VARIABLE) {
-                chipTag = createChipTag('variable', compound.metadata.content);
-            } else {
-                // Skip unknown token types
-                continue;
-            }
-
-            renderableTokens.push({
-                start: compound.start,
-                end: compound.end,
-                chipTag
-            });
-        }
-
-        // Process individual tokens that are not part of compounds
-        for (let i = 0; i < parsedTokens.length; i++) {
-            const token = parsedTokens[i];
-
-            // Skip tokens that are part of compound sequences
-            if (compoundTokenIndices.has(i)) {
-                continue;
+        // Use the shared iterator from CodeUtils
+        iterateTokenStream(parsedTokens, compounds, expression, {
+            onCompound: (compound) => {
+                let chipTag: string | undefined;
+                if (compound.tokenType === TokenType.DOCUMENT && compound.metadata.documentType) {
+                    chipTag = createChipTag(TokenType.DOCUMENT, compound.metadata.content, compound.metadata.documentType);
+                } else if (compound.tokenType === TokenType.VARIABLE) {
+                    chipTag = createChipTag(TokenType.VARIABLE, compound.metadata.content);
+                }
+
+                if (chipTag) {
+                    renderableTokens.push({
+                        start: compound.start,
+                        end: compound.end,
+                        chipTag
+                    });
+                }
+            },
+            onToken: (token, content) => {
+                const chipTag = createChipTag(token.type, content);
+                if (chipTag) {
+                    renderableTokens.push({
+                        start: token.start,
+                        end: token.end,
+                        chipTag
+                    });
+                }
             }
-
-            // Skip START_EVENT and END_EVENT tokens
-            if (token.type === TokenType.START_EVENT || token.type === TokenType.END_EVENT) {
-                continue;
-            }
-
-            // Get the actual text content for this token
-            const content = expression.slice(token.start, token.end);
-
-            let chipTag = createChipTag(token.type, content);
-
-            if (chipTag) {
-                renderableTokens.push({
-                    start: token.start,
-                    end: token.end,
-                    chipTag
-                });
-            }
-        }
+        });
 
         // If no tokens to render, return original expression
         if (renderableTokens.length === 0) {
             return expression;
         }
 
         // Sort by position in reverse order to maintain position integrity
-        // when replacing (replace from end to start)
         const sortedTokens = renderableTokens.sort((a, b) => b.start - a.start);
 
         let transformed = expression;
-
         for (const token of sortedTokens) {
-            // Replace the token range with chip tag
             transformed =
                 transformed.slice(0, token.start) +
                 token.chipTag +
                 transformed.slice(token.end);
         }
         return transformed;
+
     } catch (error) {
         console.error('Error transforming expression to markdown:', error);
-        // Return original expression if transformation fails
         return expression;
     }
 };
 
 export const hasTokens = (expression: string): boolean => {
     return /\$\{[^}]+\}/.test(expression);
-};
+};

workspaces/ballerina/ballerina-side-panel/src/components/editors/MultiModeExpressionEditor/ChipExpressionEditor/CodeUtils.ts

@@ -258,6 +258,58 @@ export const tokenField = StateField.define<TokenFieldState>({
     }
 });
 
+export const iterateTokenStream = (
+    tokens: ParsedToken[],
+    compounds: CompoundTokenSequence[],
+    content: string,
+    callbacks: {
+        onCompound: (compound: CompoundTokenSequence) => void;
+        onToken: (token: ParsedToken, text: string) => void;
+    }
+) => {
+    const docLength = content.length;
+
+    // Create a set of token indices that are part of compounds
+    const compoundTokenIndices = new Set<number>();
+    for (const compound of compounds) {
+        for (let i = compound.startIndex; i <= compound.endIndex; i++) {
+            compoundTokenIndices.add(i);
+        }
+    }
+
+    // Process compound tokens
+    for (const compound of compounds) {
+        // Validate compound range
+        if (compound.start < 0 || compound.end > docLength || compound.start >= compound.end) {
+            continue;
+        }
+        callbacks.onCompound(compound);
+    }
+
+    // Process individual tokens that are not part of compounds
+    for (let i = 0; i < tokens.length; i++) {
+        const token = tokens[i];
+
+        // Skip tokens that are part of compound sequences
+        if (compoundTokenIndices.has(i)) {
+            continue;
+        }
+
+        // Skip START_EVENT and END_EVENT tokens
+        if (token.type === TokenType.START_EVENT || token.type === TokenType.END_EVENT) {
+            continue;
+        }
+
+        // Validate token range
+        if (token.start < 0 || token.end > docLength || token.start >= token.end) {
+            continue;
+        }
+
+        const text = content.slice(token.start, token.end);
+        callbacks.onToken(token, text);
+    }
+};
+
 export const chipPlugin = ViewPlugin.fromClass(
     class {
         decorations: RangeSet<Decoration>;
@@ -274,60 +326,35 @@ export const chipPlugin = ViewPlugin.fromClass(
             }
         }
         buildDecorations(view: EditorView) {
-            const widgets = [];
+            const widgets: any[] = []; // Type as any[] to allow pushing Range<Decoration>
             const { tokens, compounds } = view.state.field(tokenField);
-            const docLength = view.state.doc.length;
-
-            // Create a set of token indices that are part of compounds
-            const compoundTokenIndices = new Set<number>();
-            for (const compound of compounds) {
-                for (let i = compound.startIndex; i <= compound.endIndex; i++) {
-                    compoundTokenIndices.add(i);
-                }
-            }
-
-            // Render compound tokens as single chips
-            for (const compound of compounds) {
-                // Validate compound range
-                if (compound.start < 0 || compound.end > docLength || compound.start >= compound.end) {
-                    continue;
-                }
-
-                const chipType: TokenType = compound.tokenType;
-                widgets.push(
-                    createChip(
-                        compound.displayText,
-                        chipType,
-                        compound.start,
-                        compound.end,
-                        view,
-                        compound.metadata
-                    ).range(compound.start, compound.end)
-                );
-            }
-
-            // Render individual tokens that are not part of compounds
-            for (let i = 0; i < tokens.length; i++) {
-                const token = tokens[i];
-
-                // Skip tokens that are part of compound sequences
-                if (compoundTokenIndices.has(i)) {
-                    continue;
-                }
-
-                // Skip START_EVENT and END_EVENT tokens
-                if (token.type === TokenType.START_EVENT || token.type === TokenType.END_EVENT) {
-                    continue;
-                }
-
-                // Validate token range
-                if (token.start < 0 || token.end > docLength || token.start >= token.end) {
-                    continue;
+            const docContent = view.state.doc.toString();
+
+            iterateTokenStream(tokens, compounds, docContent, {
+                onCompound: (compound) => {
+                    widgets.push(
+                        createChip(
+                            compound.displayText,
+                            compound.tokenType,
+                            compound.start,
+                            compound.end,
+                            view,
+                            compound.metadata
+                        ).range(compound.start, compound.end)
+                    );
+                },
+                onToken: (token, text) => {
+                    widgets.push(
+                        createChip(
+                            text,
+                            token.type,
+                            token.start,
+                            token.end,
+                            view
+                        ).range(token.start, token.end)
+                    );
                 }
-
-                const text = view.state.doc.sliceString(token.start, token.end);
-                widgets.push(createChip(text, token.type, token.start, token.end, view).range(token.start, token.end));
-            }
+            });
 
             return Decoration.set(widgets, true);
         }