basic AI integration

packages/editor/package.json

@@ -38,6 +38,8 @@
     "@typecell-org/parsers": "^0.0.3",
     "@typecell-org/frame": "^0.0.3",
     "@typecell-org/y-penpal": "^0.0.3",
+    "openai": "^4.11.1",
+    "ai": "2.2.14",
     "speakingurl": "^14.0.1",
     "classnames": "^2.3.1",
     "fractional-indexing": "^2.0.0",

packages/editor/src/app/documentRenderers/richtext/FrameHost.tsx

@@ -1,10 +1,11 @@
-import { IframeBridgeMethods } from "@typecell-org/shared";
+import { HostBridgeMethods, IframeBridgeMethods } from "@typecell-org/shared";
 import { ContainedElement, useResource } from "@typecell-org/util";
 import { PenPalProvider } from "@typecell-org/y-penpal";
 import { AsyncMethodReturns, connectToChild } from "penpal";
 import { useRef } from "react";
 import * as awarenessProtocol from "y-protocols/awareness";
 import { parseIdentifier } from "../../../identifiers";
+import { queryOpenAI } from "../../../integrations/ai/openai";
 import { DocumentResource } from "../../../store/DocumentResource";
 import { DocumentResourceModelProvider } from "../../../store/DocumentResourceModelProvider";
 import { SessionStore } from "../../../store/local/SessionStore";
@@ -64,7 +65,7 @@ export function FrameHost(props: {
       { provider: DocumentResourceModelProvider; forwarder: ModelForwarder }
     >();
 
-    const methods = {
+    const methods: HostBridgeMethods = {
       processYjsMessage: async (message: ArrayBuffer) => {
         provider.onMessage(message, "penpal");
       },
@@ -110,6 +111,7 @@ export function FrameHost(props: {
         moduleManager.forwarder.dispose();
         moduleManagers.delete(identifierStr);
       },
+      queryLLM: queryOpenAI,
     };
 
     const iframe = document.createElement("iframe");

packages/editor/src/integrations/ai/openai.ts

@@ -0,0 +1,43 @@
+import { OpenAIStream, StreamingTextResponse } from "ai";
+import { OpenAI } from "openai";
+
+export async function queryOpenAI(parameters: {
+  messages: OpenAI.Chat.ChatCompletionCreateParams["messages"];
+  functions?: OpenAI.Chat.ChatCompletionCreateParams["functions"];
+  function_call?: OpenAI.Chat.ChatCompletionCreateParams["function_call"];
+}) {
+  // get key from localstorage
+  let key = localStorage.getItem("oai-key");
+  if (!key) {
+    key = prompt(
+      "Please enter your OpenAI key (not shared with TypeCell, stored in your browser):",
+    );
+    if (!key) {
+      return {
+        status: "error",
+        error: "no-key",
+      } as const;
+    }
+    localStorage.setItem("oai-key", key);
+  }
+
+  const openai = new OpenAI({
+    apiKey: key,
+    // this should be ok as we are not exposing any keys
+    dangerouslyAllowBrowser: true,
+  });
+
+  const response = await openai.chat.completions.create({
+    model: "gpt-4",
+    stream: true,
+    ...parameters,
+  });
+  const stream = OpenAIStream(response);
+  // Respond with the stream
+  const ret = new StreamingTextResponse(stream);
+  const data = await ret.text();
+  return {
+    status: "ok",
+    result: data,
+  } as const;
+}

packages/engine/src/executor.ts

@@ -13,6 +13,7 @@ async function resolveDependencyArray(
   userDisposes: Array<() => void>,
 ) {
   const runContext = {
+    context,
     onDispose: (disposer: () => void) => {
       userDisposes.push(() => {
         try {
@@ -47,6 +48,7 @@ async function resolveDependencyArray(
 }
 
 export type RunContext = {
+  context: TypeCellContext<any>;
   onDispose: (disposer: () => void) => void;
 };
 
@@ -120,6 +122,7 @@ export async function runModule(
       disposeEveryRun.push(hooks.disposeAll);
       let executionPromise: Promise<any>;
       try {
+        console.log("execute", mod.factoryFunction + "");
         executionPromise = mod.factoryFunction.apply(
           undefined,
           argsToCallFunctionWith,

packages/engine/src/modules.ts

@@ -28,10 +28,11 @@ export function getModulesFromWrappedPatchedTypeCellFunction(
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   caller: () => any,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  scope: any
+  scope: any,
 ): Module[] {
   const modules: Module[] = [];
   const define = createDefine(modules);
+  console.log("evaluate (module)", caller + "");
   caller.apply({ ...scope, define });
   return modules;
 }
@@ -43,10 +44,11 @@ export function getModulesFromWrappedPatchedTypeCellFunction(
 export function getModulesFromPatchedTypeCellCode(
   code: string,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  scope: any
+  scope: any,
 ): Module[] {
   const modules: Module[] = [];
   const define = createDefine(modules);
+  console.log("evaluate (module)", code);
   // eslint-disable-next-line
   const f = new Function(code);
   f.apply({ ...scope, define });
@@ -57,7 +59,7 @@ function createDefine(modules: Module[]) {
   return function typeCellDefine(
     moduleNameOrDependencyArray: string | string[],
     dependencyArrayOrFactoryFunction: string[] | Function,
-    factoryFunction?: Function
+    factoryFunction?: Function,
   ) {
     const moduleName: string | typeof unnamedModule =
       typeof moduleNameOrDependencyArray === "string"
@@ -118,7 +120,7 @@ export function getPatchedTypeCellCode(compiledCode: string, scope: any) {
 
   totalCode = totalCode.replace(
     /^\s*(define\((".*", )?\[.*\], )function/gm,
-    "$1async function"
+    "$1async function",
   ); // TODO: remove await?
 
   return totalCode;

packages/frame/package.json

@@ -3,6 +3,7 @@
   "version": "0.0.3",
   "private": true,
   "dependencies": {
+    "@atlaskit/form": "^8.11.8",
     "@blocknote/core": "^0.9.3",
     "@blocknote/react": "^0.9.3",
     "@typecell-org/util": "^0.0.3",
@@ -14,12 +15,12 @@
     "@floating-ui/react": "^0.25.1",
     "@syncedstore/yjs-reactive-bindings": "^0.5.1",
     "lodash.memoize": "^4.1.2",
-    "mobx-utils": "^6.0.8",
     "localforage": "^1.10.0",
     "lz-string": "^1.4.4",
     "monaco-editor": "^0.35.0",
     "mobx": "^6.2.0",
     "mobx-react-lite": "^3.2.0",
+    "mobx-utils": "^6.0.8",
     "prosemirror-model": "^1.19.3",
     "prosemirror-view": "^1.31.7",
     "prosemirror-state": "^1.4.3",
@@ -33,7 +34,8 @@
     "typescript": "5.0.4",
     "vscode-lib": "^0.1.2",
     "y-protocols": "^1.0.5",
-    "yjs": "^13.6.4"
+    "yjs": "^13.6.4",
+    "y-prosemirror": "^1.0.20"
   },
   "devDependencies": {
     "cross-fetch": "^4.0.0",
@@ -45,7 +47,9 @@
     "@vitest/coverage-v8": "^0.33.0",
     "@vitejs/plugin-react": "^4.1.0",
     "@types/prettier": "^3.0.0",
-    "chai": "^4.3.7"
+    "chai": "^4.3.7",
+    "openai": "^4.11.1",
+    "ai": "2.2.14"
   },
   "type": "module",
   "source": "src/index.ts",

packages/frame/src/EditorStore.ts

@@ -23,11 +23,17 @@ export class EditorStore {
   public executionHost: LocalExecutionHost | undefined;
   public topLevelBlocks: any;
 
+  public readonly customBlocks = new Map<string, any>();
+  public readonly blockSettings = new Map<string, any>();
+
   constructor() {
     makeObservable(this, {
       customBlocks: observable.shallow,
-      add: action,
-      delete: action,
+      addCustomBlock: action,
+      deleteCustomBlock: action,
+      blockSettings: observable.shallow,
+      addBlockSettings: action,
+      deleteBlockSettings: action,
       topLevelBlocks: observable.ref,
     });
 
@@ -45,12 +51,10 @@ export class EditorStore {
     });
   }
 
-  customBlocks = new Map<string, any>();
-
   /**
    * Add a custom block (slash menu command) to the editor
    */
-  public add(config: any) {
+  public addCustomBlock(config: any) {
     if (this.customBlocks.has(config.id)) {
       // already has block with this id, maybe loop of documents?
       return;
@@ -61,10 +65,28 @@ export class EditorStore {
   /**
    * Remove a custom block (slash menu command) from the editor
    */
-  public delete(config: any) {
+  public deleteCustomBlock(config: any) {
     this.customBlocks.delete(config.id);
   }
 
+  /**
+   * Add a block settings (block settings menu) to the editor
+   */
+  public addBlockSettings(config: any) {
+    if (this.blockSettings.has(config.id)) {
+      // already has block with this id, maybe loop of documents?
+      return;
+    }
+    this.blockSettings.set(config.id, config);
+  }
+
+  /**
+   * Remove block settings (block settings menu) from the editor
+   */
+  public deleteBlockSettings(config: any) {
+    this.blockSettings.delete(config.id);
+  }
+
   /**
    * EXPERIMENTAL
    * @internal
@@ -180,11 +202,11 @@ class TypeCellBlock {
       runInAction(() => {
         this.block = newBlock;
 
-        if (newBlock.props.storage !== JSON.stringify(this.storage)) {
-          if (newBlock.props.storage) {
+        if ((newBlock.props as any).storage !== JSON.stringify(this.storage)) {
+          if (newBlock.props as any) {
             try {
               console.log("update cell storage");
-              this.storage = JSON.parse(newBlock.props.storage) || {};
+              this.storage = JSON.parse((newBlock.props as any).storage) || {};
             } catch (e) {
               console.error(e);
             }
@@ -221,7 +243,7 @@ class TypeCellBlock {
           editor.updateBlock(this.block, {
             props: {
               storage: val,
-            },
+            } as any,
           });
         }
       },

packages/frame/src/Frame.tsx

@@ -32,16 +32,18 @@ import styles from "./Frame.module.css";
 import { RichTextContext } from "./RichTextContext";
 import { MonacoCodeBlock } from "./codeblocks/MonacoCodeBlock";
 import SourceModelCompiler from "./runtime/compiler/SourceModelCompiler";
+import { setMonacoDefaults } from "./runtime/editor";
 import { MonacoContext } from "./runtime/editor/MonacoContext";
-import { ExecutionHost } from "./runtime/executor/executionHosts/ExecutionHost";
 import LocalExecutionHost from "./runtime/executor/executionHosts/local/LocalExecutionHost";
 
-import { setMonacoDefaults } from "./runtime/editor";
-
 import { variables } from "@typecell-org/util";
 import { RiCodeSSlashFill } from "react-icons/ri";
+import { VscWand } from "react-icons/vsc";
 import { EditorStore } from "./EditorStore";
 import { MonacoColorManager } from "./MonacoColorManager";
+
+import { getAICode } from "./ai/ai";
+import { applyChanges } from "./ai/applyChanges";
 import { MonacoInlineCode } from "./codeblocks/MonacoInlineCode";
 import monacoStyles from "./codeblocks/MonacoSelection.module.css";
 import { setupTypecellHelperTypeResolver } from "./runtime/editor/languages/typescript/TypeCellHelperTypeResolver";
@@ -94,18 +96,20 @@ const originalItems = [
   ...getDefaultReactSlashMenuItems(),
   {
     name: "Code block",
-    execute: (editor: any) =>
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    execute: (editor: BlockNoteEditor<any>) =>
       insertOrUpdateBlock(editor, {
         type: "codeblock",
-      } as any),
+      }),
     aliases: ["code"],
     hint: "Add a live code block",
     group: "Code",
     icon: <RiCodeSSlashFill size={18} />,
   },
   {
     name: "Inline",
-    execute: (editor: any) => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    execute: (editor: BlockNoteEditor<any>) => {
       // state.tr.replaceSelectionWith(dinoType.create({type}))
       const node = editor._tiptapEditor.schema.node(
         "inlineCode",
@@ -274,12 +278,13 @@ export const Frame: React.FC<Props> = observer((props) => {
         resolver.resolveImport,
       );
 
-      const newExecutionHost: ExecutionHost = new LocalExecutionHost(
+      const newExecutionHost = new LocalExecutionHost(
         props.documentIdString,
         newCompiler,
         monaco,
         newEngine,
       );
+
       return [
         { newCompiler, newExecutionHost },
         () => {
@@ -290,64 +295,94 @@ export const Frame: React.FC<Props> = observer((props) => {
     [props.documentIdString, monaco],
   );
 
-  console.log("size", editorStore.current.customBlocks.size);
   slashMenuItems.splice(
     originalItems.length,
     slashMenuItems.length,
-    ...[...editorStore.current.customBlocks.values()].map((data: any) => {
+    {
+      name: "AI",
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      execute: async (editor: BlockNoteEditor<any>) => {
+        const p = prompt("What would you like TypeCell AI to do?");
+        if (!p) {
+          return;
+        }
+
+        const commands = await getAICode(
+          p,
+          props.documentIdString,
+          tools.newExecutionHost,
+          editor,
+          editorStore.current,
+          // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          connectionMethods.current!.queryLLM,
+        );
+
+        // TODO: we should validate the commands before applying them
+        applyChanges(
+          commands,
+          document.ydoc.getXmlFragment("doc"),
+          document.awareness,
+        );
+      },
+      aliases: ["ai", "wizard", "openai", "llm"],
+      hint: "Prompt your TypeCell AI assistant",
+      group: "Code",
+      icon: <VscWand size={18} />,
+    },
+    ...[...editorStore.current.customBlocks.values()].map((data) => {
       console.log("update blocks");
       return {
         name: data.name,
-        execute: (editor: any) => {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        execute: (editor: BlockNoteEditor<any>) => {
           const origVarName = variables.toCamelCaseVariableName(data.name);
           let varName = origVarName;
           let i = 0;
           // eslint-disable-next-line no-constant-condition
           while (true) {
             // append _1, _2, _3, ... to the variable name until it is unique
 
-            if (
-              // eslint-disable-next-line @typescript-eslint/no-explicit-any
-              (
-                tools.newExecutionHost.engine.observableContext
-                  .rawContext as any
-              )[varName] === undefined
-            ) {
+            const context =
+              tools.newExecutionHost.engine.observableContext.rawContext;
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            if ((context as any)[varName] === undefined) {
               break;
             }
             i++;
             varName = origVarName + "_" + i;
           }
 
-          insertOrUpdateBlock(
-            editor as any,
-            {
-              type: "codeblock",
-              props: {
-                language: "typescript",
-                // moduleName: moduleName,
-                // key,
-                storage: "",
-              },
-              content: `// @default-collapsed
+          const settingsPart = data.settings
+            ? `
+typecell.editor.registerBlockSettings({
+  content: (visible: boolean) => (
+    <typecell.AutoForm
+      inputObject={doc}
+      fields={${JSON.stringify(data.settings, undefined, 2)}}
+      visible={visible}
+    />
+  ),
+});
+`
+            : "";
+          insertOrUpdateBlock(editor, {
+            type: "codeblock",
+            props: {
+              language: "typescript",
+              storage: "",
+            },
+            content: `// @default-collapsed
 import * as doc from "${data.documentId}";
 
-export let ${varName} = doc.${data.blockVariable};
+export let ${varName} = doc.${data.blockExport};
 export let ${varName}Scope = doc;
-
+${settingsPart}
 export default ${varName};
 `,
-            } as any,
-          );
+          });
         },
-        // execute: (editor) =>
-        //   insertOrUpdateBlock(editor, {
-        //     type: data[0],
-        //   }),
-        // aliases: [data[0]],
-        // hint: "Add a " + data[0],
         group: "Custom",
-      } as any;
+      };
     }),
   );
 
@@ -400,6 +435,7 @@ export default ${varName};
   });
 
   if (editorStore.current.editor !== editor) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     editorStore.current.editor = editor as any;
   }
 
@@ -412,6 +448,7 @@ export default ${varName};
       <MonacoContext.Provider value={{ monaco }}>
         <RichTextContext.Provider
           value={{
+            editorStore: editorStore.current,
             executionHost: tools.newExecutionHost,
             compiler: tools.newCompiler,
             documentId: props.documentIdString,

packages/frame/src/RichTextContext.ts

@@ -1,12 +1,16 @@
 import { createContext } from "react";
+import { EditorStore } from "./EditorStore";
 import SourceModelCompiler from "./runtime/compiler/SourceModelCompiler";
 import { ExecutionHost } from "./runtime/executor/executionHosts/ExecutionHost";
 
 export const RichTextContext = createContext<{
+  editorStore: EditorStore;
   executionHost: ExecutionHost;
   compiler: SourceModelCompiler;
   documentId: string;
 }>({
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  editorStore: undefined as any,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   executionHost: undefined as any,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any

packages/frame/src/ai/ai.ts

@@ -0,0 +1,376 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+// import LocalExecutionHost from "../../../runtime/executor/executionHosts/local/LocalExecutionHost"
+import "@blocknote/core/style.css";
+import * as mobx from "mobx";
+import * as monaco from "monaco-editor";
+import type openai from "openai";
+
+import { BlockNoteEditor } from "@blocknote/core";
+import { HostBridgeMethods } from "@typecell-org/shared";
+import { uri } from "vscode-lib";
+import { EditorStore } from "../EditorStore";
+import { ExecutionHost } from "../runtime/executor/executionHosts/ExecutionHost";
+import { trimmedStringify } from "./trimmedStringify";
+
+const TYPECELL_PROMPT = `
+You're a smart AI assistant for TypeCell: a rich text document tool that also supports interactive Code Blocks written in Typescript. 
+
+TypeCell Documents look like this:
+- Documents consists of a list of blocks (e.g.: headings, paragraphs, code blocks), Notion style. Code Blocks are unique to TypeCell and execute live, as-you type.
+
+TypeCell Code Blocks works as follows:
+- Code Blocks can export variables using the javascript / typescript \`export\` syntax. These variables are shown as the output of the cell.
+- The exported variables by a Code Block are available in other cells, under the \`$\` variable. e.g.: \`$.exportedVariableFromOtherCell\`
+- Different cells MUST NOT output variables with the same name, because then they would collide under the \`$\` variable.
+- When the exports of one Code Block change, other cells that depend on those exports, update live, automatically.
+- React / JSX components will be displayed automatically. E.g.: \`export let component = <div>hello world</div>\` will display a div with hello world.
+- Note that exported functions are not called automatically. They'll simply become a callable variable under the $ scope. This means simply exporting a function and not calling it anywhere is not helpful
+
+Example document:
+
+[
+  {
+    id: "block-1",
+    type: "codeblock",
+    content: "export let name = 'James';",
+  },
+  {
+    id: "block-2",
+    type: "codeblock",
+    content: "export let nameLength = $.name.length; // updates reactively based on the $.name export from block-1",
+  },
+  {
+    id: "block-3",
+    type: "codeblock",
+    content: "// This uses the exported \`name\` from code block 1, using the TypeCell \`$.name\` syntax, and shows the capitalized name using React
+    export let capitalized = <div>{$.name.toUpperCase()}</div>",
+  }
+]
+
+The runtime data of this would be: 
+
+{ name: "James", nameLength: 5, capitalized: "[REACTELEMENT]"}
+
+This is the type of a document:
+
+type Block = {
+  id: string;
+  type: "paragraph" | "heading" | "codeblock";
+  content?: string;
+  children?: Block[];
+};
+
+export type Document = Block[];
+
+Example prompts:
+- If the user would ask you to update the name in the document, you would issue an Update operation to block-1.
+- If the user would ask you to add a button to prompt for a name, you would issue an Add operation for a new codeblock with code \`export default <button onClick={() => $.name = prompt('what's your name?'}>Change name</button>\`
+- If the user would ask you to output the name in reverse, you would issue an Add operation with code \`export let reverseName = $.name.split('').reverse().join('');\`
+
+NEVER write code that depends on and updates the same variable, as that would cause a loop. You can directly modify (mutate) variables. So don't do this:
+
+$.complexObject = { ...$.complexObject, newProperty: 5 };
+
+but instead:
+
+$.complexObject.newProperty = 5;
+`;
+
+export async function getAICode(
+  prompt: string,
+  documentId: string,
+  executionHost: ExecutionHost,
+  editor: BlockNoteEditor<any>,
+  editorStore: EditorStore,
+  queryLLM: HostBridgeMethods["queryLLM"],
+) {
+  const blocks = editor.topLevelBlocks;
+
+  let blockContexts: any[] = [];
+  const iterateBlocks = (blocks: any[]) => {
+    for (const block of blocks) {
+      const b = editorStore.getBlock(block.id);
+      if (b?.context?.default) {
+        blockContexts.push(b.context.default);
+      }
+      iterateBlocks(block.children);
+    }
+  };
+  iterateBlocks(blocks);
+
+  blockContexts = blockContexts.map((output) =>
+    Object.fromEntries(
+      Object.getOwnPropertyNames(output).map((key) => [
+        key,
+        mobx.toJS(output[key]),
+      ]),
+    ),
+  );
+
+  const tmpModel = monaco.editor.createModel(
+    "",
+    "typescript",
+    uri.URI.parse("file:///tmp.tsx"),
+  );
+
+  tmpModel.setValue(`import * as React from "react";
+  import * as $ from "!${documentId}";
+  // expands object types one level deep
+type Expand<T> = T extends infer O ? { [K in keyof O]: O[K] extends { Key: React.Key | null } ? "[REACT]" : O[K] } : never;
+
+// expands object types recursively
+type ExpandRecursively<T> = T extends object
+  ? T extends (...args: any[]) => any
+    ? T
+    : T extends infer O
+    ? {
+        [K in keyof O]: O[K] extends { key: React.Key }
+          ? "[REACT ELEMENT]"
+          : ExpandRecursively<O[K]>;
+      }
+    : never
+  : T;
+  
+  // ? T extends infer O ? { [K in keyof O]: ExpandRecursively<O[K]> } : never
+  type ContextType = ExpandRecursively<typeof $>;`);
+
+  const worker = await monaco.languages.typescript.getTypeScriptWorker();
+
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  const ts = (await worker(tmpModel.uri))!;
+  const pos =
+    tmpModel.getValue().length - "pe = ExpandRecursively<typeof $>;".length;
+  // const def = await ts.getDefinitionAtPosition(tmpModel.uri.toString(), pos);
+  const def2 = await ts.getQuickInfoAtPosition(tmpModel.uri.toString(), pos);
+
+  const contextType = def2.displayParts.map((x: any) => x.text).join("");
+
+  // const def3 = await ts.get(tmpModel.uri.toString(), pos, {});
+  tmpModel.dispose();
+
+  /*
+  // const models = monaco.editor.getModels();
+  // const typeCellModels = models.filter((m) =>
+  //   m.uri.path.startsWith("/!typecell:typecell.org"),
+  // );
+
+  
+  const codeInfoPromises = typeCellModels.map(async (m) => {
+    const code = await compile(m, monaco);
+    const output = await executionHost.outputs.get(m.uri.toString())?.value;
+
+    let data: any;
+    if (output) {
+      const outputJS = Object.fromEntries(
+        Object.getOwnPropertyNames(output).map((key) => [
+          key,
+          mobx.toJS(output[key]),
+        ]),
+      );
+      data = JSON.parse(customStringify(outputJS));
+      // console.log(data);
+    }
+    const path = m.uri.path.split("/"); // /!typecell:typecell.org/dVVAYmvBaeQdE/c58863ef-2f82-4fd7-ab0c-f1f760eb9578.cell.tsx"
+    const blockId = path[path.length - 1].replace(".cell.tsx", "");
+    const imported = !blocks.find((b) => b.id === blockId);
+
+    const ret: CodeBlockRuntimeInfo = {
+      // code: imported ? m.getValue() : undefined,
+      types: code.types,
+      blockId,
+      data,
+      ...(imported
+        ? { documentId: path[path.length - 2]!, imported, code: m.getValue() }
+        : { imported }),
+    };
+    return ret;
+  });
+  let codeInfos = await Promise.all(codeInfoPromises);
+  codeInfos = codeInfos.filter((x) => !!x.imported);*/
+
+  const context = executionHost.engine.observableContext.rawContext as any;
+
+  let outputJS = Object.fromEntries(
+    Object.getOwnPropertyNames(context).map((key) => [
+      key,
+      mobx.toJS(context[key]),
+    ]),
+  );
+  outputJS = JSON.parse(trimmedStringify(outputJS));
+
+  function cleanBlock(block: any) {
+    if (!block.content?.length && !block.children?.length) {
+      return undefined;
+    }
+    delete block.props;
+    if (block.children) {
+      block.children = block.children.map(cleanBlock);
+    }
+    if (Array.isArray(block.content)) {
+      block.content = block.content.map((x: any) => x.text).join("");
+    }
+    return block;
+  }
+
+  const sanitized = blocks.map(cleanBlock).filter((x) => !!x);
+  const contextInfo =
+    contextType.replace("type ContextType = ", "const $: ") +
+    " = " +
+    JSON.stringify(outputJS);
+
+  const blockContextInfo = blockContexts.length
+    ? `typecell.editor.findBlocks = (predicate: (context) => boolean) {
+        return (${JSON.stringify(blockContexts)}).find(predicate);
+    }`
+    : undefined;
+
+  const messages: openai.Chat.ChatCompletionCreateParams["messages"] = [
+    {
+      role: "system",
+      content: TYPECELL_PROMPT,
+    },
+    {
+      role: "user",
+      content: `This is my document data: 
+"""${JSON.stringify(sanitized)}"""`,
+    },
+    {
+      role: "user",
+      content:
+        "This is the type and runtime data available under the reactive $ variable for read / write access. If you need to change / read some information from the live document, it's likely you need to access it from here using $.<variable name> \n" +
+        contextInfo +
+        (blockContextInfo
+          ? "\n" +
+            `We also have this function "typecell.editor.findBlocks" to extract runtime data from blocks \n` +
+            blockContextInfo
+          : ""),
+    },
+    {
+      role: "system",
+      content: `You are an AI assistant helping user to modify his document. This means changes can either be code related (in that case, you'll need to add / modify Code Blocks), 
+        or not at all (in which case you'll need to add / modify regular blocks), or a mix of both.`,
+    },
+    {
+      role: "user",
+      content: prompt,
+    },
+  ];
+
+  // Ask OpenAI for a streaming chat completion given the prompt
+  const response = await queryLLM({
+    messages,
+    functions: [
+      {
+        name: "updateDocument",
+        description: "Update the document with operations",
+        parameters: {
+          $schema: "http://json-schema.org/draft-07/schema#",
+          type: "object",
+          required: ["operations"],
+          properties: {
+            operations: {
+              type: "array",
+              items: {
+                oneOf: [
+                  {
+                    type: "object",
+                    properties: {
+                      explanation: {
+                        type: "string",
+                        description:
+                          "explanation of why this block was deleted (your reasoning as AI agent)",
+                      },
+                      type: {
+                        type: "string",
+                        enum: ["delete"],
+                        description:
+                          "Operation to delete a block in the document",
+                      },
+                      id: {
+                        type: "string",
+                        description: "id of block to delete",
+                      },
+                    },
+                    required: ["type", "id"],
+                    additionalProperties: false,
+                  },
+                  {
+                    type: "object",
+                    properties: {
+                      explanation: {
+                        type: "string",
+                        description:
+                          "explanation of why this block was updated (your reasoning as AI agent)",
+                      },
+                      type: {
+                        type: "string",
+                        enum: ["update"],
+                        description:
+                          "Operation to update a block in the document",
+                      },
+                      id: {
+                        type: "string",
+                        description: "id of block to delete",
+                      },
+                      content: {
+                        type: "string",
+                        description: "new content of block",
+                      },
+                    },
+                    required: ["type", "id", "content"],
+                    additionalProperties: false,
+                  },
+                  {
+                    type: "object",
+                    properties: {
+                      explanation: {
+                        type: "string",
+                        description:
+                          "explanation of why this block was added (your reasoning as AI agent)",
+                      },
+                      type: {
+                        type: "string",
+                        enum: ["add"],
+                        description:
+                          "Operation to insert a new block in the document",
+                      },
+                      afterId: {
+                        type: "string",
+                        description:
+                          "id of block after which to insert a new block in the document",
+                      },
+                      content: {
+                        type: "string",
+                        description: "content of new block",
+                      },
+                      blockType: {
+                        type: "string",
+                        enum: ["codeblock", "paragraph", "heading"],
+                        description: "type of new block",
+                      },
+                    },
+                    required: ["afterId", "type", "content", "blockType"],
+                    additionalProperties: false,
+                  },
+                ],
+              },
+            },
+          },
+        },
+      },
+    ],
+    function_call: {
+      name: "updateDocument",
+    },
+  });
+
+  console.log(messages);
+
+  if (response.status === "ok") {
+    const data = JSON.parse(response.result);
+    return JSON.parse(data.function_call.arguments).operations;
+  } else {
+    console.error("queryLLM error", response.error);
+  }
+  return undefined;
+}

packages/frame/src/ai/applyChanges.ts

@@ -0,0 +1,172 @@
+import { error, uniqueId } from "@typecell-org/util";
+import * as ypm from "y-prosemirror";
+import { Awareness } from "y-protocols/awareness";
+import * as Y from "yjs";
+import { BlockOperation, OperationsResponse } from "./types";
+import { getYjsDiffs } from "./yjsDiff";
+
+function findBlock(id: string, data: Y.XmlFragment) {
+  const node = data
+    .createTreeWalker(
+      (el) => el instanceof Y.XmlElement && el.getAttribute("id") === id,
+    )
+    .next();
+  if (node.done) {
+    return undefined;
+  }
+  return node.value as Y.XmlElement;
+}
+
+function findParentIndex(node: Y.XmlFragment) {
+  const parent = node.parent as Y.XmlElement;
+  for (let i = 0; i < parent.length; i++) {
+    if (parent.get(i) === node) {
+      return i;
+    }
+  }
+  throw new Error("not found");
+}
+
+function updateState(
+  awareness: Awareness,
+  head: Y.RelativePosition,
+  anchor: Y.RelativePosition,
+) {
+  // const initial = !awareness.states.has(99);
+  awareness.states.set(99, {
+    user: {
+      name: "@AI",
+      color: "#94FADB",
+    },
+    cursor: {
+      anchor,
+      head,
+    },
+  });
+
+  awareness.emit("change", [
+    {
+      added: 0,
+      updated: 1,
+      removed: 0,
+    },
+    origin,
+  ]);
+}
+
+export async function applyChange(
+  op: BlockOperation,
+  data: Y.XmlFragment,
+  awareness: Awareness,
+) {
+  const transact = (op: () => void) => {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    Y.transact(data.doc!, op, ypm.ySyncPluginKey);
+  };
+  if (op.type === "add") {
+    const node = findBlock(op.afterId, data);
+    if (!node) {
+      throw new Error("Block not found");
+    }
+    const newElement = new Y.XmlElement("blockContainer");
+    const child = new Y.XmlElement(op.blockType);
+    child.setAttribute("id", uniqueId.generateId("block"));
+    const yText = new Y.XmlText();
+    child.insert(0, [yText]);
+    newElement.insert(0, [child]);
+
+    transact(() => {
+      (node.parent as Y.XmlElement).insertAfter(node, [newElement]);
+    });
+    // start typing text content
+    for (let i = 0; i < op.content.length; i++) {
+      const start = Y.createRelativePositionFromTypeIndex(yText, i);
+      const end = Y.createRelativePositionFromTypeIndex(yText, i);
+      updateState(awareness, start, end);
+
+      transact(() => {
+        yText.insert(i, op.content[i]);
+      });
+      await new Promise((resolve) => setTimeout(resolve, 20));
+    }
+  } else if (op.type === "delete") {
+    const node = findBlock(op.id, data);
+    if (!node) {
+      throw new Error("Block not found");
+    }
+    const blockNode = node.firstChild as Y.XmlElement;
+    const yText = blockNode.firstChild as Y.XmlText;
+
+    const start = Y.createRelativePositionFromTypeIndex(yText, 0);
+    const end = Y.createRelativePositionFromTypeIndex(yText, yText.length - 1);
+
+    updateState(awareness, start, end);
+
+    await new Promise((resolve) => setTimeout(resolve, 200));
+
+    transact(() => {
+      (node.parent as Y.XmlElement).delete(findParentIndex(node), 1);
+    });
+    await new Promise((resolve) => setTimeout(resolve, 20));
+  } else if (op.type === "update") {
+    const node = findBlock(op.id, data);
+    if (!node) {
+      throw new Error("Block not found");
+    }
+
+    // let gptCode = "\n" + gptCell.code + "\n";
+    // gptCode = gptCode.replaceAll("import React from 'react';\n", "");
+    // gptCode = gptCode.replaceAll("import * as React from 'react';\n", "");
+    // console.log("diffs", cell.code.toJSON(), gptCode);
+    const blockNode = node.firstChild as Y.XmlElement;
+    const yText = blockNode.firstChild as Y.XmlText;
+    const steps = getYjsDiffs(yText, op.content);
+    for (const step of steps) {
+      if (step.type === "insert") {
+        for (let i = 0; i < step.text.length; i++) {
+          const start = Y.createRelativePositionFromTypeIndex(
+            yText,
+            step.from + i,
+          );
+          const end = Y.createRelativePositionFromTypeIndex(
+            yText,
+            step.from + i,
+          );
+          updateState(awareness, start, end);
+
+          transact(() => {
+            yText.insert(step.from + i, step.text[i]);
+          });
+          await new Promise((resolve) => setTimeout(resolve, 20));
+        }
+      } else if (step.type === "delete") {
+        const start = Y.createRelativePositionFromTypeIndex(yText, step.from);
+        const end = Y.createRelativePositionFromTypeIndex(
+          yText,
+          step.from + step.length,
+        );
+        updateState(awareness, start, end);
+        await new Promise((resolve) => setTimeout(resolve, 200));
+        transact(() => {
+          yText.delete(step.from, step.length);
+        });
+        await new Promise((resolve) => setTimeout(resolve, 20));
+      }
+    }
+  } else {
+    throw new error.UnreachableCaseError(op);
+  }
+}
+
+export async function applyChanges(
+  commands: OperationsResponse,
+  fragment: Y.XmlFragment,
+  awareness: Awareness,
+) {
+  const doc = new Y.Doc();
+
+  for (const op of commands) {
+    await applyChange(op, fragment, awareness);
+  }
+  return doc;
+}

packages/frame/src/ai/trimmedStringify.ts

@@ -0,0 +1,141 @@
+import React from "react";
+
+// quickly generated with chatgpt:
+// "create a json stringify alternative that can trim long fields / deeply nested objects,
+// and serializes property getters"
+
+// https://chat.openai.com/share/693c349f-3f74-4913-8f1d-ba4477f93e87
+
+type Serializable =
+  | string
+  | number
+  | boolean
+  | null
+  | { [key: string]: Serializable }
+  | Serializable[];
+
+interface QueueItem {
+  obj: Serializable;
+  path: (string | number)[];
+}
+
+/**
+ * a stringify function that trims long fields / deeply nested objects, and serializes property getters
+ */
+export function trimmedStringify(obj: Serializable, budget = 1000): string {
+  const seen = new Set<Serializable>();
+  const queue: QueueItem[] = [{ obj, path: [] }];
+  const output: Serializable = Array.isArray(obj) ? [] : {};
+
+  while (queue.length > 0 && budget > 0) {
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const { obj: currentObj, path } = queue.shift()!;
+
+    if (typeof currentObj !== "object" || currentObj === null) {
+      continue;
+    }
+
+    // Handle circular references
+    if (seen.has(currentObj)) {
+      setByPath(output, path, "[CIRCULAR]");
+      continue;
+    }
+    seen.add(currentObj);
+
+    for (const key in currentObj) {
+      if (budget <= 0) {
+        break;
+      }
+
+      const value = (currentObj as any)[key];
+      const newPath = path.concat(key);
+
+      if (typeof value === "string") {
+        if (value.length <= budget) {
+          setByPath(output, newPath, value);
+          budget -= value.length;
+        } else {
+          setByPath(
+            output,
+            newPath,
+            value.substring(0, budget - "[TRIMMED]".length) + "[TRIMMED]",
+          );
+          budget = 0;
+        }
+      } else if (typeof value === "object" && value !== null) {
+        if (Array.isArray(value)) {
+          const newValue: Serializable[] = [];
+          setByPath(output, newPath, newValue);
+          if (JSON.stringify(value).length > budget) {
+            newValue.push("[TRIMMEDARRAY]");
+            budget -= "[TRIMMEDARRAY]".length;
+          } else {
+            queue.push({ obj: value, path: newPath });
+          }
+        } else if (React.isValidElement(value)) {
+          setByPath(output, newPath, "[REACTELEMENT]");
+        } else {
+          const newValue: Serializable = {};
+          setByPath(output, newPath, newValue);
+          if (JSON.stringify(value).length > budget) {
+            for (const prop in newValue) {
+              delete newValue[prop];
+            }
+            setByPath(output, newPath, "[TRIMMEDOBJECT]");
+            budget -= "[TRIMMEDOBJECT]".length;
+          } else {
+            queue.push({ obj: value, path: newPath });
+          }
+        }
+      } else {
+        setByPath(output, newPath, value);
+      }
+    }
+  }
+
+  return JSON.stringify(output);
+
+  function setByPath(
+    obj: Serializable,
+    path: (string | number)[],
+    value: Serializable,
+  ): void {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    let current: any = obj; // We'll refine this with type assertions as we traverse
+    for (let i = 0; i < path.length - 1; i++) {
+      if (typeof current[path[i]] === "undefined") {
+        current[path[i]] = typeof path[i + 1] === "number" ? [] : {};
+      }
+      current = current[path[i]];
+    }
+    current[path[path.length - 1]] = value;
+  }
+}
+
+// Example usage:
+// const obj = {
+//   name: "John",
+//   details: {
+//     age: 25,
+//     address: {
+//       street: "123 Main St",
+//       city: "Anytown",
+//       state: "CA",
+//       country: {
+//         name: "USA",
+//         code: "US",
+//         continent: {
+//           name: "North America",
+//           code: "NA",
+//         },
+//       },
+//     },
+//   },
+//   hobbies: ["reading", "traveling", "swimming", "hiking", "cycling"],
+//   bio: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
+//   get fullName() {
+//     return this.name + " Doe";
+//   },
+// };
+
+// console.log(customStringify(obj, 200));

packages/frame/src/ai/types.ts

@@ -0,0 +1,91 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/**
+ * Operation to the document
+ *
+ * Block Id `id` parameters MUST be part of the document the user is editing (NOT a block from an imported library)
+ */
+export type BlockOperation =
+  | {
+      type: "delete";
+      id: string;
+    }
+  | {
+      type: "update";
+      id: string;
+      content: string;
+    }
+  | {
+      afterId: string;
+      type: "add";
+      content: string;
+      blockType: "codeblock" | "paragraph" | "heading";
+    };
+
+export type OperationsResponse = BlockOperation[];
+
+export const OUTPUT_TYPES = `/**
+* Operation to the document
+*
+* Block Id \`id\` parameters MUST be part of the document the user is editing (NOT a block from an imported library)
+*/
+type BlockOperation =
+ | {
+     type: "delete";
+     id: string;
+   }
+ | {
+     type: "update";
+     id: string;
+     content: string;
+   }
+ | {
+     afterId: string;
+     type: "add";
+     content: string;
+     blockType: "codeblock" | "paragraph" | "heading";
+   };
+
+type response = BlockOperation[];`;
+
+type Block = {
+  id: string;
+  type: "paragraph" | "heading" | "codeblock";
+  content?: string;
+  children?: Block[];
+};
+
+export type Document = Block[];
+
+/**
+ * Runtime information about a code block of the main document
+ * The code itself is not included (it's in the Block.id with the corresponding blockId)
+ */
+type MainCodeBlockRuntimeInfo = {
+  imported: false;
+  blockId: string;
+  // .d.ts TypeScript types of values exported by this block
+  types: string;
+  // the runtime values exported by this block. Data can be trimmed for brevity
+  data: any;
+};
+
+/**
+ * Runtime + code information of code blocks imported from other documents
+ */
+type ImportedCodeBlockRuntimeInfo = {
+  imported: true;
+  /**
+   * Because we don't pass the entire document this code is imported from, we need to pass the code of this code block
+   */
+  code: string;
+  // .d.ts TypeScript types of values exported by this block
+  types: string;
+  documentId: string;
+  blockId: string;
+  // the runtime values exported by this block. Data can be trimmed for brevity
+  data: any;
+};
+
+export type CodeBlockRuntimeInfo =
+  | MainCodeBlockRuntimeInfo
+  | ImportedCodeBlockRuntimeInfo;

packages/frame/src/ai/yjsDiff.test.ts

@@ -0,0 +1,60 @@
+/**
+ * @vitest-environment jsdom
+ */
+
+import { describe, expect, it } from "vitest";
+import * as Y from "yjs";
+import { getYjsDiffs } from "./yjsDiff";
+
+describe("diffYjs", () => {
+  it("basic replace", () => {
+    const doc = new Y.Doc();
+
+    // const text = new Y.Text("hello world");
+    const text = doc.getText("text");
+    text.insert(0, "hello world");
+    getYjsDiffs(text, "hello there world");
+    expect(text.toJSON()).toEqual("hello there world");
+  });
+
+  it("delete", () => {
+    const doc = new Y.Doc();
+
+    // const text = new Y.Text("hello world");
+    const text = doc.getText("text");
+    text.insert(0, "hello there world");
+    getYjsDiffs(text, "hello world");
+    expect(text.toJSON()).toEqual("hello world");
+  });
+
+  it("insert and delete", () => {
+    const doc = new Y.Doc();
+
+    // const text = new Y.Text("hello world");
+    const text = doc.getText("text");
+    text.insert(0, "hello there world");
+    getYjsDiffs(text, "hell crazy world. How are you?");
+    expect(text.toJSON()).toEqual("hell crazy world. How are you?");
+  });
+
+  it("advanced", () => {
+    const orig = `// This generates an array of numbers 1 through 10
+    export let numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];`;
+
+    const newText = ` 
+
+    // This cell exports an array of numbers 1 through 9
+    
+    export let numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9];
+    
+    `;
+
+    const doc = new Y.Doc();
+
+    // const text = new Y.Text("hello world");
+    const text = doc.getText("text");
+    text.insert(0, orig);
+    getYjsDiffs(text, newText);
+    expect(text.toJSON()).toEqual(newText);
+  });
+});

packages/frame/src/ai/yjsDiff.ts

@@ -0,0 +1,75 @@
+import * as Y from "yjs";
+
+import diff_match_patch from "../runtime/editor/prettier/diff";
+import { trimPatch } from "../runtime/editor/prettier/trimPatch";
+
+const dmp = new diff_match_patch();
+
+type Step =
+  | {
+      type: "insert";
+      text: string;
+      from: number;
+    }
+  | {
+      type: "delete";
+      from: number;
+      length: number;
+    };
+
+export function getYjsDiffs(
+  existing: Y.Text,
+  newText: string,
+  execute = false,
+) {
+  const steps: Step[] = [];
+
+  const diffs = dmp.diff_main(existing.toJSON(), newText);
+  const patches = dmp.patch_make(diffs);
+
+  //   let posDiff = 0;
+  for (const patch of patches) {
+    trimPatch(patch);
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const startPos = patch.start1!; // + posDiff;
+    // posDiff += patch.length1 - patch.length2;
+
+    let tempLengths = 0;
+    for (const diff of patch.diffs) {
+      const type = diff[0];
+      const text = diff[1];
+
+      // type 0: keep, type 1: insert, type -1: delete
+      if (type === 0) {
+        tempLengths += text.length;
+      } else if (type === 1) {
+        // newText += text;
+        const actionStart = startPos + tempLengths;
+        steps.push({
+          type: "insert",
+          text,
+          from: actionStart,
+          // action: () => existing.insert(actionStart, text),
+        });
+        if (execute) {
+          existing.insert(actionStart, text);
+        }
+        tempLengths += text.length;
+      } else {
+        // tempLengths -= text.length;
+        // posDiff -= patch.length1;
+        const actionStart = startPos + tempLengths;
+
+        steps.push({
+          type: "delete",
+          from: actionStart,
+          length: text.length,
+        });
+        if (execute) {
+          existing.delete(actionStart, text.length);
+        }
+      }
+    }
+  }
+  return steps;
+}

packages/frame/src/codeblocks/MonacoCodeBlock.tsx

@@ -85,6 +85,7 @@ export const MonacoCodeBlock = createTipTapBlock<"codeblock", any>({
         // class: styles.blockContent,
         "data-content-type": this.name,
       }),
+      0,
     ];
   },
 

packages/frame/src/codeblocks/MonacoElement.tsx

@@ -10,7 +10,11 @@ import React, {
   useRef,
   useState,
 } from "react";
-import { VscChevronDown, VscChevronRight } from "react-icons/vsc";
+import {
+  VscChevronDown,
+  VscChevronRight,
+  VscSettingsGear,
+} from "react-icons/vsc";
 
 import {
   autoUpdate,
@@ -261,30 +265,47 @@ const MonacoBlockElement = (
   const [codeVisible, setCodeVisible] = useState(
     () => props.node.textContent.startsWith("// @default-collapsed") === false,
   );
+  const [settingsVisible, setSettingsVisible] = useState(false);
 
   const context = useContext(RichTextContext);
 
+  const settings = context.editorStore.blockSettings.get(
+    props.model.uri.toString(),
+  );
   return (
     <div
       contentEditable={false}
       className={[
         styles.codeCell,
         codeVisible ? styles.expanded : styles.collapsed,
       ].join(" ")}>
-      {codeVisible ? (
-        <VscChevronDown
-          title="Show / hide code"
-          className={styles.codeCellSideIcon}
-          onClick={() => setCodeVisible(false)}
-        />
-      ) : (
-        <VscChevronRight
-          title="Show / hide code"
-          className={styles.codeCellSideIcon}
-          onClick={() => setCodeVisible(true)}
-        />
-      )}
-      {}
+      <div
+        style={{
+          display: "flex",
+          flexDirection: "column",
+          alignItems: "center",
+        }}>
+        {codeVisible ? (
+          <VscChevronDown
+            title="Show / hide code"
+            className={styles.codeCellSideIcon}
+            onClick={() => setCodeVisible(false)}
+          />
+        ) : (
+          <VscChevronRight
+            title="Show / hide code"
+            className={styles.codeCellSideIcon}
+            onClick={() => setCodeVisible(true)}
+          />
+        )}
+        {settings && (
+          <VscSettingsGear
+            size={12}
+            className={styles.codeCellSideIcon}
+            onClick={() => setSettingsVisible(!settingsVisible)}
+          />
+        )}
+      </div>
       <div className={styles.codeCellContent}>
         {codeVisible && (
           <div className={styles.codeCellCode}>
@@ -309,6 +330,7 @@ const MonacoBlockElement = (
             },
           )}
         </div>
+        {settings && <div>{settings.content(settingsVisible)}</div>}
       </div>
     </div>
   );

packages/frame/src/runtime/editor/compilerOptions.ts

@@ -14,6 +14,8 @@ export function getDefaultSandboxCompilerOptions(
   >,
 ) {
   const settings: CompilerOptions = {
+    // used so that ts.getQuickInfoAtPosition doesn't truncate too soon
+    noErrorTruncation: true,
     noImplicitAny: true,
     strictNullChecks: !config.useJavaScript,
     strictFunctionTypes: true,

packages/frame/src/runtime/editor/prettier/diffToMonacoTextEdits.ts

@@ -2,40 +2,10 @@
 /* eslint-disable @typescript-eslint/no-non-null-assertion */
 import type * as monaco from "monaco-editor";
 import diff_match_patch from "./diff.js";
+import { trimPatch } from "./trimPatch.js";
 
 const dmp = new diff_match_patch();
 
-/**
- * Trim type-0 diffs from a diff_match_patch patch. 0 indicates "keep", so is not really a diff
- */
-function trimPatch(patch: any) {
-  // head
-  if (patch.diffs[0][0] === 0) {
-    const len = patch.diffs[0][1].length;
-
-    // adjust patch params
-    patch.start1 += len;
-    patch.length1 -= len;
-    patch.start2 += len;
-    patch.length2 -= len;
-
-    // remove diff
-    patch.diffs.shift();
-  }
-  // tail
-  if (patch.diffs[patch.diffs.length - 1][0] === 0) {
-    const len = patch.diffs[patch.diffs.length - 1][1].length;
-
-    // adjust patch params
-    patch.length1 -= len;
-    patch.length2 -= len;
-
-    // remove diff
-    patch.diffs.pop();
-  }
-  return patch;
-}
-
 /**
  * This calculates a list of Monaco TextEdit objects, that represent the transformation from
  * model.getValue() to v2.
@@ -55,7 +25,7 @@ export function diffToMonacoTextEdits(model: monaco.editor.IModel, v2: string) {
     trimPatch(patch);
     const startPos = model.getPositionAt(patch.start1! + posDiff);
     const endPos = model.getPositionAt(
-      patch.start1! + patch.length1! + posDiff
+      patch.start1! + patch.length1! + posDiff,
     );
     const range: monaco.IRange = {
       startColumn: startPos.column,

packages/frame/src/runtime/editor/prettier/trimPatch.ts

@@ -0,0 +1,30 @@
+/**
+ * Trim type-0 diffs from a diff_match_patch patch. 0 indicates "keep", so is not really a diff
+ */
+export function trimPatch(patch: any) {
+  // head
+  if (patch.diffs[0][0] === 0) {
+    const len = patch.diffs[0][1].length;
+
+    // adjust patch params
+    patch.start1 += len;
+    patch.length1 -= len;
+    patch.start2 += len;
+    patch.length2 -= len;
+
+    // remove diff
+    patch.diffs.shift();
+  }
+  // tail
+  if (patch.diffs[patch.diffs.length - 1][0] === 0) {
+    const len = patch.diffs[patch.diffs.length - 1][1].length;
+
+    // adjust patch params
+    patch.length1 -= len;
+    patch.length2 -= len;
+
+    // remove diff
+    patch.diffs.pop();
+  }
+  return patch;
+}

packages/frame/src/runtime/executor/components/ModelOutput.ts

@@ -11,7 +11,9 @@ export class ModelOutput extends lifecycle.Disposable {
   private autorunDisposer: (() => void) | undefined;
 
   public value: any = undefined;
-  public _defaultValue: any = {};
+  public _defaultValue = {
+    value: {} as any,
+  };
   public typeVisualizers = observable.map<
     string,
     {
@@ -21,6 +23,7 @@ export class ModelOutput extends lifecycle.Disposable {
 
   constructor(private context: any) {
     super();
+
     makeObservable(this, {
       typeVisualizers: observable.ref,
       value: observable.ref,
@@ -70,13 +73,14 @@ export class ModelOutput extends lifecycle.Disposable {
         }
       }
 
-      this._defaultValue = newValue.default;
+      // hacky nesting to make sure our customAnnotation (for react elements) is used
+      this._defaultValue = { value: newValue.default };
 
       if (changed) {
         if (Object.hasOwn(newValue, "default")) {
           Object.defineProperty(newValue, "default", {
             get: () => {
-              return this.defaultValue;
+              return this.defaultValue.value;
             },
           });
         }

packages/frame/src/runtime/executor/executionHosts/local/LocalExecutionHost.tsx

@@ -35,7 +35,10 @@ export default class LocalExecutionHost
     //     );
     //   })
     // );
-    this.engine.registerModelProvider(compileEngine);
+
+    if (!window.location.hash.includes("noRun")) {
+      this.engine.registerModelProvider(compileEngine);
+    }
 
     const visualizerExtension = this._register(
       new VisualizerExtension(compileEngine, documentId, monacoInstance),

packages/frame/src/runtime/executor/lib/autoForm/FormField.tsx

@@ -20,7 +20,7 @@ export const FormField = observer(
       [key: string]: unknown;
     };
     fieldKey: Key;
-    modelPath: string;
+    // modelPath: string;
     value: string | undefined;
     setValue: (value: string | undefined) => void;
   }) => {
@@ -57,6 +57,8 @@ export const FormField = observer(
 
     let inputField: React.ReactNode = <div>Unsupported type</div>;
 
+    const realShowCode = showCode || !canUseInput;
+
     if (canUseInput) {
       const valueType =
         currentParsedBinding === undefined
@@ -112,10 +114,10 @@ export const FormField = observer(
         {({ fieldProps, error }) => (
           <Fragment>
             <div style={{ display: "flex" }}>
-              {showCode ? (
+              {realShowCode ? (
                 <MonacoEdit
                   value={props.value || "export default"}
-                  documentid={props.modelPath}
+                  documentid={"TODO"}
                   onChange={(newValue) => {
                     if (!newValue || newValue.trim() === "export default") {
                       props.setValue(undefined);
@@ -151,14 +153,14 @@ export const FormField = observer(
                   <DropdownItemRadio
                     id="value"
                     onClick={() => setShowCode(false)}
-                    isSelected={!showCode}
+                    isSelected={!realShowCode}
                     isDisabled={!canUseInput}>
                     Value view
                   </DropdownItemRadio>
                   <DropdownItemRadio
                     id="code"
                     onClick={() => setShowCode(true)}
-                    isSelected={showCode}>
+                    isSelected={realShowCode}>
                     Code view
                   </DropdownItemRadio>
                 </DropdownItemRadioGroup>

packages/frame/src/runtime/executor/lib/autoForm/MonacoEdit.tsx

@@ -11,7 +11,8 @@ type Props = {
 };
 
 const MonacoEdit: React.FC<Props> = observer((props) => {
-  console.log(props);
+  // console.log(props);
+
   const uri = useMemo(
     () => monaco.Uri.parse(`${props.documentid}.edit.${Math.random()}.tsx`),
     [props.documentid],

packages/frame/src/runtime/executor/lib/autoForm/index.tsx

@@ -1,5 +1,6 @@
 import Form, { FormSection } from "@atlaskit/form";
 import { observer } from "mobx-react-lite";
+import React from "react";
 import { FormField } from "./FormField";
 import { Settings } from "./types";
 
@@ -54,7 +55,7 @@ export const AutoForm = observer(
                       key={input}
                       inputObject={props.inputObject as any}
                       fieldKey={input}
-                      modelPath={props.modelPath}
+                      // modelPath={props.modelPath}
                       value={props.settings[input]}
                       setValue={(value: string | undefined) => {
                         props.setSetting(input, value);

packages/frame/src/runtime/executor/lib/exports.tsx

@@ -21,7 +21,11 @@ export default function getExposeGlobalVariables(
     // registerPlugin: (config: { name: string }) => {
     //   return config.name;
     // },
-    registerBlock: (config: { name: string; blockExport: string }) => {
+    registerBlock: (config: {
+      name: string;
+      blockExport: string;
+      settings?: Record<string, boolean>;
+    }) => {
       // TODO: this logic should be part of CodeModel / BasicCodeModel
       const id = forModelList[forModelList.length - 1].path;
       const parts = decodeURIComponent(id.replace("file:///", "")).split("/");
@@ -35,11 +39,27 @@ export default function getExposeGlobalVariables(
         documentId,
       };
       console.log("ADD BLOCK", completeConfig.id);
-      editorStore.add(completeConfig);
+      editorStore.addCustomBlock(completeConfig);
 
       runContext.onDispose(() => {
         console.log("REMOVE BLOCK", completeConfig.id);
-        editorStore.delete(completeConfig);
+        editorStore.deleteCustomBlock(completeConfig);
+      });
+    },
+    registerBlockSettings: (config: any) => {
+      // TODO: this logic should be part of CodeModel / BasicCodeModel
+      const id = forModelList[forModelList.length - 1].uri;
+
+      const completeConfig: any = {
+        ...config,
+        id: id.toString(),
+      };
+      // console.log("ADD BLOCK", completeConfig.id);
+      editorStore.addBlockSettings(completeConfig);
+
+      runContext.onDispose(() => {
+        // console.log("REMOVE BLOCK", completeConfig.id);
+        editorStore.deleteBlockSettings(completeConfig);
       });
     },
     /**
@@ -125,7 +145,7 @@ export default function getExposeGlobalVariables(
     },
   };
   return {
-    memoize: (func: (...args: any[]) => any) => {
+    memoize: <T extends (...args: any[]) => any>(func: T): T => {
       const wrapped = async function (this: any, ...args: any[]) {
         const ret = await func.apply(this, args);
         // if (typeof ret === "object") {
@@ -135,7 +155,7 @@ export default function getExposeGlobalVariables(
       };
       return memoize(wrapped, (args) => {
         return JSON.stringify(args);
-      });
+      }) as any as T;
     },
     // routing,
     // // DocumentView,
@@ -163,7 +183,9 @@ export default function getExposeGlobalVariables(
           [key: string]: unknown;
         },
       >(
-        props: Exclude<AutoFormProps<T>, "settings" | "setSettings">,
+        props: Exclude<AutoFormProps<T>, "settings" | "setSettings"> & {
+          visible: boolean;
+        },
       ) => {
         const storage = editor.currentBlock.storage;
 
@@ -182,7 +204,7 @@ export default function getExposeGlobalVariables(
             const func = new Function("$target", "$", sanitizedCode);
             return () => {
               console.log("eval", key, code);
-              func(props.inputObject, props.inputObject); // TODO
+              func(props.inputObject, runContext.context.context);
             };
           });
         }, [props.inputObject, storage.settings]);
@@ -212,6 +234,10 @@ export default function getExposeGlobalVariables(
           });
         }, [props.fields, createFunctionTransformer]);
 
+        if (!props.visible) {
+          return <></>;
+        }
+
         return (
           <AutoForm
             {...props}

packages/server/package.json

@@ -34,6 +34,7 @@
     "start": "node dist/src/index.js",
     "start:supabase": "./supabase.sh start",
     "stop:supabase": "./supabase.sh stop",
+    "dump": "./supabase.sh db dump -f export.sql --db-url=postgresql://postgres:postgres@localhost:54322 --data-only",
     "clean": "rimraf dist && rimraf types",
     "dev": "MODE=development vite-node src/index.ts",
     "build": "npm run clean && tsc -p tsconfig.json",

packages/shared/src/frameInterop/HostBridgeMethods.ts

@@ -18,4 +18,27 @@ export type HostBridgeMethods = {
    * Function for y-penpal
    */
   processYjsMessage: (message: Uint8Array) => Promise<void>;
+
+  /**
+   * Function to query LLM (openai)
+   * Executed in host, so that the key can be stored in localstorage and
+   * cannot be accessed by user-scripts
+   */
+  queryLLM: (parameters: {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    messages: any[];
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    functions?: any[];
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    function_call?: any;
+  }) => Promise<
+    | {
+        status: "ok";
+        result: string;
+      }
+    | {
+        status: "error";
+        error: string;
+      }
+  >;
 };