[BI Data Mapper] Group by clause support for the Data mapper in frontend UI

workspaces/ballerina/ballerina-core/src/interfaces/data-mapper.ts

@@ -57,9 +57,10 @@ export enum IntermediateClauseType {
     LET = "let",
     WHERE = "where",
     FROM = "from",
-    ORDER_BY = "order by",
+    ORDER_BY = "order-by",
     LIMIT = "limit",
     JOIN = "join",
+    GROUP_BY = "group-by"
 }
 
 export enum ResultClauseType {
@@ -95,6 +96,7 @@ export interface IOType {
     defaultValue?: unknown;
     optional?: boolean;
     isFocused?: boolean;
+    isSeq?: boolean;
     isRecursive?: boolean;
     isDeepNested?: boolean;
     ref?: string;
@@ -141,6 +143,7 @@ export interface DMModel {
     triggerRefresh?: boolean;
     traversingRoot?: string;
     focusInputRootMap?: Record<string, string>;
+    groupById?: string;
 }
 
 export interface ModelState {
@@ -175,6 +178,7 @@ export interface IOTypeField {
     optional?: boolean;
     ref?: string;
     focusExpression?: string;
+    isSeq?: boolean;
     typeInfo?: TypeInfo;
 }
 
@@ -192,7 +196,7 @@ export interface Query {
     output: string,
     inputs: string[];
     diagnostics?: DMDiagnostic[];
-    fromClause: FromClause;
+    fromClause: IntermediateClause;
     intermediateClauses?: IntermediateClause[];
     resultClause: ResultClause;
 }

workspaces/ballerina/ballerina-extension/src/rpc-managers/data-mapper/utils.ts

@@ -19,9 +19,6 @@ import {
     CodeData,
     ELineRange,
     Flow,
-    AllDataMapperSourceRequest,
-    DataMapperSourceRequest,
-    DataMapperSourceResponse,
     NodePosition,
     ProjectStructureArtifactResponse,
     TextEdit,
@@ -34,7 +31,8 @@ import {
     IORoot,
     ExpandModelOptions,
     ExpandedDMModel,
-    MACHINE_VIEW
+    MACHINE_VIEW,
+    IntermediateClauseType
 } from "@wso2/ballerina-core";
 import { updateSourceCode, UpdateSourceCodeRequest } from "../../utils";
 import { StateMachine, updateDataMapperView } from "../../stateMachine";
@@ -598,16 +596,24 @@ function processArray(
     let fieldId = generateFieldId(parentId, member.name);
 
     let isFocused = false;
+    let isGroupByIdUpdated = false;
+    const prevGroupById = model.groupById;
+
     if (model.focusInputs) {
         const focusMember = model.focusInputs[parentId];
         if (focusMember) {
             member = focusMember;
             parentId = member.name;
             fieldId = member.name;
             isFocused = true;
+            model.focusInputRootMap[fieldId] = model.traversingRoot;
 
-            if (model.traversingRoot){
-                model.focusInputRootMap[parentId] = model.traversingRoot;
+            if(member.isSeq && model.query!.fromClause.properties.name === fieldId){
+                const groupByClause = model.query!.intermediateClauses?.find(clause => clause.type === IntermediateClauseType.GROUP_BY);
+                if(groupByClause){
+                    model.groupById = groupByClause.properties.name;
+                    isGroupByIdUpdated = true;
+                }
             }
         }
     }
@@ -625,6 +631,10 @@ function processArray(
 
     const typeSpecificProps = processTypeKind(member, parentId, model, visitedRefs);
 
+    if(isGroupByIdUpdated){
+        model.groupById = prevGroupById;
+    }
+
     return {
         ...ioType,
         ...typeSpecificProps
@@ -718,13 +728,31 @@ function processTypeFields(
     if (!type.fields) { return []; }
 
     return type.fields.map(field => {
-        const fieldId = generateFieldId(parentId, field.name!);
+        let fieldId = generateFieldId(parentId, field.name!);
+
+        let isFocused = false;
+        let isSeq = !!model.groupById;
+        if (model.focusInputs) {
+            const focusMember = model.focusInputs[fieldId];
+            if (focusMember) {
+                field = focusMember;
+                fieldId = field.name;
+                isFocused = true;
+                model.focusInputRootMap[fieldId] = model.traversingRoot;
+                if (fieldId === model.groupById){
+                    isSeq = false;
+                }
+            }
+        }
+
         const ioType: IOType = {
             id: fieldId,
             name: field.name,
             displayName: field.displayName,
             typeName: field.typeName,
             kind: field.kind,
+            ...(isFocused && { isFocused }),
+            ...(isSeq && { isSeq }),
             ...(field.optional !== undefined && { optional: field.optional }),
             ...(field.typeInfo && { typeInfo: field.typeInfo })
         };

workspaces/ballerina/ballerina-visualizer/src/views/DataMapper/DataMapperView.tsx

@@ -43,7 +43,9 @@ import {
     MACHINE_VIEW,
     VisualizerLocation,
     DeleteClauseRequest,
-    IORoot
+    IORoot,
+    IntermediateClauseType,
+    TriggerKind
 } from "@wso2/ballerina-core";
 import { CompletionItem, ProgressIndicator } from "@wso2/ui-toolkit";
 import { useRpcContext } from "@wso2/ballerina-rpc-client";
@@ -368,9 +370,14 @@ export function DataMapperView(props: DataMapperProps) {
                 targetField: targetField,
                 index: index
             });
-            return position;
+            if (position) {
+                return position;
+            } else {
+                throw new Error("Clause position not found");
+            }
         } catch (error) {
             console.error(error);
+            return { line: 0, offset: 0  };
         }
     }
 
@@ -540,11 +547,45 @@ export function DataMapperView(props: DataMapperProps) {
         parentField.isDeepNested = false;
     }
 
+    const genUniqueName = async (name: string, viewId: string): Promise<string> => {
+        const { property } = await rpcClient.getDataMapperRpcClient().getProperty({
+            filePath,
+            codedata: viewState.codedata,
+            targetField: viewId
+        })
+
+        if (!property?.codedata?.lineRange?.startLine) {
+            console.error("Failed to get start line for generating unique name");
+            return name;
+        }
+
+        const completions = await rpcClient.getBIDiagramRpcClient().getDataMapperCompletions({
+            filePath,
+            context: {
+                expression: "",
+                startLine: property.codedata.lineRange.startLine,
+                lineOffset: 0,
+                offset: 0,
+                codedata: viewState.codedata,
+                property: property
+            },
+            completionContext: {
+                triggerKind: TriggerKind.INVOKED
+            }
+        });
+
+        let i = 2;
+        let uniqueName = name;
+        while (completions.some(c => c.insertText === uniqueName)) {
+            uniqueName = name + (i++);
+        }
 
+        return uniqueName;
+    };
 
     const onDMClose = () => {
         onClose ? onClose() : rpcClient.getVisualizerRpcClient()?.goBack();
-    }
+    };
 
     const onDMRefresh = async () => {
         try {
@@ -573,7 +614,7 @@ export function DataMapperView(props: DataMapperProps) {
         };
 
         rpcClient.getVisualizerRpcClient().openView({ type: EVENT_TYPE.OPEN_VIEW, location: context });
-    }
+    };
 
 
     useEffect(() => {
@@ -621,7 +662,7 @@ export function DataMapperView(props: DataMapperProps) {
                         property: property
                     },
                     completionContext: {
-                        triggerKind: triggerCharacter ? 2 : 1,
+                        triggerKind: triggerCharacter ? TriggerKind.TRIGGER_CHARACTER : TriggerKind.INVOKED,
                         triggerCharacter: triggerCharacter as TriggerCharacter
                     }
                 });
@@ -708,6 +749,7 @@ export function DataMapperView(props: DataMapperProps) {
                             mapWithTransformFn={mapWithTransformFn}
                             goToFunction={goToFunction}
                             enrichChildFields={enrichChildFields}
+                            genUniqueName={genUniqueName}
                             undoRedoGroup={undoRedoGroup}
                             expressionBar={{
                                 completions: filteredCompletions,
@@ -727,7 +769,13 @@ export function DataMapperView(props: DataMapperProps) {
 };
 
 const getModelSignature = (model: DMModel | ExpandedDMModel): ModelSignature => ({
-    inputs: [...model.inputs.map(i => i.name), ...(model.query?.inputs || [])],
+    inputs: [...model.inputs.map(i => i.name),
+    ...(model.query?.inputs || []),
+    ...(model.query?.intermediateClauses
+        ?.filter((clause) => (clause.type === IntermediateClauseType.LET || clause.type === IntermediateClauseType.GROUP_BY))
+        .map(clause => `${clause.properties.type} ${clause.properties.name} ${clause.properties.expression}`)
+        || [])
+    ],
     output: model.output.name,
     subMappings: model.subMappings?.map(s => (s as IORoot | IOType).name) || [],
     refs: 'refs' in model ? JSON.stringify(model.refs) : ''

workspaces/ballerina/data-mapper/src/components/DataMapper/DataMapperEditor.tsx

@@ -144,6 +144,7 @@ export function DataMapperEditor(props: DataMapperEditorProps) {
         mapWithTransformFn,
         goToFunction,
         enrichChildFields,
+        genUniqueName,
         undoRedoGroup
     } = props;
     const {
@@ -236,10 +237,12 @@ export function DataMapperEditor(props: DataMapperEditorProps) {
                 convertToQuery,
                 deleteMapping,
                 deleteSubMapping,
+                addClauses,
                 mapWithCustomFn,
                 mapWithTransformFn,
                 goToFunction,
-                enrichChildFields
+                enrichChildFields,
+                genUniqueName
             );
 
             const ioNodeInitVisitor = new IONodeInitVisitor(context);
@@ -364,6 +367,7 @@ export function DataMapperEditor(props: DataMapperEditorProps) {
                                 deleteClause={deleteClause}
                                 getClausePosition={getClausePosition}
                                 generateForm={generateForm}
+                                genUniqueName={genUniqueName}
                             />
                         )}
                     </>

workspaces/ballerina/data-mapper/src/components/DataMapper/SidePanel/QueryClauses/ClauseEditor.tsx

@@ -42,12 +42,13 @@ export function ClauseEditor(props: ClauseEditorProps) {
 
     const [clauseType, setClauseType] = React.useState<string>(_clauseType ?? IntermediateClauseType.WHERE);
     const clauseTypeItems: OptionProps[] = [
-        { content: "condition", value: IntermediateClauseType.WHERE },
-        { content: "local variable", value: IntermediateClauseType.LET },
-        { content: "sort by", value: IntermediateClauseType.ORDER_BY },
-        { content: "limit", value: IntermediateClauseType.LIMIT },
-        { content: "from", value: IntermediateClauseType.FROM },
-        { content: "join", value: IntermediateClauseType.JOIN },
+        { content: "Condition", value: IntermediateClauseType.WHERE },
+        { content: "Local variable", value: IntermediateClauseType.LET },
+        { content: "Sort by", value: IntermediateClauseType.ORDER_BY },
+        { content: "Limit", value: IntermediateClauseType.LIMIT },
+        { content: "From", value: IntermediateClauseType.FROM },
+        { content: "Join", value: IntermediateClauseType.JOIN },
+        { content: "Group by", value: IntermediateClauseType.GROUP_BY }
     ]
 
     const nameField: DMFormField = {
@@ -76,11 +77,15 @@ export function ClauseEditor(props: ClauseEditorProps) {
 
     const expressionField: DMFormField = {
         key: "expression",
-        label: clauseType === IntermediateClauseType.JOIN ? "Join With Collection" : "Expression",
+        label: clauseType === IntermediateClauseType.JOIN ? "Join With Collection" :
+            clauseType === IntermediateClauseType.GROUP_BY ? "Grouping Key" :
+                "Expression",
         type: "EXPRESSION",
         optional: false,
         editable: true,
-        documentation: clauseType === IntermediateClauseType.JOIN ? "Collection to be joined" : "Enter the expression of the clause",
+        documentation: clauseType === IntermediateClauseType.JOIN ? "Collection to be joined" :
+            clauseType === IntermediateClauseType.GROUP_BY ? "Enter the grouping key expression" :
+                "Enter the expression of the clause",
         value: clauseProps?.expression ?? "",
         valueTypeConstraint: "Global",
         enabled: true,
@@ -129,10 +134,6 @@ export function ClauseEditor(props: ClauseEditorProps) {
             type: clauseType as IntermediateClauseType,
             properties: data as IntermediateClauseProps
         };
-        if (clauseType === IntermediateClauseType.JOIN) {
-            clause.properties.type = "var";
-            clause.properties.isOuter = false;
-        }
         onSubmit(clause);
     }
 
@@ -151,6 +152,8 @@ export function ClauseEditor(props: ClauseEditorProps) {
                 return [expressionField, orderField];
             case IntermediateClauseType.JOIN:
                 return [expressionField, nameField, lhsExpressionField, rhsExpressionField];
+            case IntermediateClauseType.GROUP_BY:
+                return [expressionField];
             default:
                 return [expressionField];
         }

workspaces/ballerina/data-mapper/src/components/DataMapper/SidePanel/QueryClauses/ClausesPanel.tsx

@@ -23,7 +23,7 @@ import { useDMQueryClausesPanelStore } from "../../../../store/store";
 import { AddButton, ClauseItem } from "./ClauseItem";
 import { ClauseEditor } from "./ClauseEditor";
 import { ClauseItemListContainer } from "./styles";
-import { DMFormProps, IntermediateClause, LinePosition, Query } from "@wso2/ballerina-core";
+import { DMFormProps, IntermediateClause, IntermediateClauseType, LinePosition, Query } from "@wso2/ballerina-core";
 
 export interface ClausesPanelProps {
     query: Query;
@@ -32,12 +32,13 @@ export interface ClausesPanelProps {
     deleteClause: (targetField: string, index: number) => Promise<void>;
     getClausePosition: (targetField: string, index: number) => Promise<LinePosition>;
     generateForm: (formProps: DMFormProps) => JSX.Element;
+    genUniqueName: (name: string, viewId: string) => Promise<string>;
 }
 
 export function ClausesPanel(props: ClausesPanelProps) {
     const { isQueryClausesPanelOpen, setIsQueryClausesPanelOpen } = useDMQueryClausesPanelStore();
     const { clauseToAdd, setClauseToAdd } = useDMQueryClausesPanelStore.getState();
-    const { query, targetField, addClauses, deleteClause, getClausePosition, generateForm } = props;
+    const { query, targetField, addClauses, deleteClause, getClausePosition, generateForm , genUniqueName} = props;
 
     const [adding, setAdding] = React.useState<number>();
     const [editing, setEditing] = React.useState<number>();
@@ -46,8 +47,20 @@ export function ClausesPanel(props: ClausesPanelProps) {
 
     const clauses = query?.intermediateClauses || [];
 
+    const fillDefaults = async (clause: IntermediateClause) => {
+        const clauseType = clause.type;
+        if (clauseType === IntermediateClauseType.JOIN) {
+            clause.properties.type = "var";
+            clause.properties.isOuter = false;
+        } else if (clauseType === IntermediateClauseType.GROUP_BY) {
+            clause.properties.type = "var";
+            clause.properties.name = await genUniqueName(clause.properties.expression.split('.').pop(), targetField);
+        }
+    };
+
     const setClauses = async (clause: IntermediateClause, isNew: boolean, index: number) => {
         setSaving(index);
+        await fillDefaults(clause);
         await addClauses(clause, targetField, isNew, index);
         setSaving(undefined);
     }