Fix images and multimodality.

Author: mglambda

chars/joshu/config.json

@@ -14,7 +14,9 @@
 args = p.parse_args()
 
 # get a ghostbox
-box = ghostbox.from_generic(character_folder="art_critic")
+box = ghostbox.from_generic(character_folder="art_critic",
+                            stderr=False,
+                            quiet=True)
 
 # let's make sure the images exist
 # ghostbox will not raise an error if they don't

examples/image-compare/main.py

@@ -0,0 +1,2 @@
+You are a game master in a role playing game.
+You tell a story collaboratively with a user, who is playing as {{chat_user}}.

examples/mini-adventure/game_master/system_msg

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# This is an example of a very basic interaction loop
+# it probably won't feel good, since it blocks and doesn't use streaming
+# but we want to keep it simple
+import ghostbox, time, random
+
+# the generic adapter will work with anything that supports the OAI API
+box = ghostbox.from_generic(
+    character_folder="game_master",  # see below
+    stderr=False,  # since this is a CLI program, we don't want clutter
+    quiet=True,  # we do printing and tts ourselves
+    tts=True,  # this means responses will be spoken automatically
+    tts_model="kokoro",  # kokoro is nice because it's small and good
+    tts_voice="bm_daniel",  # daniel is real GM material
+)
+
+if name := input("What is your cool adventurer name?\nName: "):
+    print(f"Welcome, {name}! A game master will be with you shortly...")
+else:
+    name = "Drizzt Do'Urden"
+    print("Better sharpen your scimitars...")
+
+# this will make {{chat_user}} expand to whatever the user just typed
+box.set_vars({"chat_user": name})
+
+print(
+    box.text(
+        "Come up with an adventure scenario and give an introduction to the player."
+    )
+)
+
+# we start conservative, but the adventure will get wilder as we go on
+current_temperature, escalation_factor = 0.3, 0.05
+while True:
+    user_msg = input("Your response (q to quit): ")
+    box.tts_stop()  # users usually like it when the tts shuts up after they hit enter
+
+    if user_msg == "q":
+        print(
+            box.text(
+                "{{chat_user}} will quit the game now. Please conclude the adventure and write a proper goodbye."
+            )
+        )
+        break
+
+    with box.options(
+        temperature=current_temperature,  # this changes every loop iteration
+        max_length=100
+        + 10
+        * random.randint(
+            -3, 3
+        ),  # keep it from talking for too long, but give some variety
+    ):
+        print(box.text(user_msg))
+
+    current_temperature = min(current_temperature + escalation_factor, 1.3)
+
+time.sleep(10)  # give time to finish the speech

examples/mini-adventure/main.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# This example shows how to use pydantic classes to get structured output from an LLM
+# You could also use json schemas, but I recommend against it.
+# Pydantic is quite wonderful.
+from pydantic import BaseModel
+from typing import *
+import ghostbox, json
+
+box = ghostbox.from_generic(character_folder="ghost-writer")
+
+
+# this is the type for the object that we will let the LLM create
+# how we name things here really matters
+class BlogPost(BaseModel):
+    title: str
+    content: str
+    tags: List[str]
+
+
+post = box.new(
+    BlogPost,  # this tells ghostbox and the backend what the structure should be
+    "Write an extremely argumentative post about how an overabundance of busking is ruining berlin.",
+)  # the prompt will provide context for filling in the python object
+print(json.dumps(post.model_dump(), indent=4))

examples/structured-output.py

@@ -7,8 +7,15 @@ class Story(object):
 
     data: List[ChatMessage] = [] 
 
-    def addUserText(self, w: str, image_id:Optional[int]=None, **kwargs) -> None:
-        new_data = ChatMessage(role = "user", content = w, image_id=image_id, **kwargs)
+    def addUserText(self, w: str, image_context: Dict[int, ImageRef]={}, **kwargs) -> None:
+        """Adds a user message to the story.
+        :param w: The user's prompt or message as plaintext.
+        :param images: A list of 0 or more images to include with the message. The images, confusingly, may be http URLs, filenames, or binary data.
+        """
+        if image_context == {}:
+            new_data = ChatMessage(role = "user", content = w, **kwargs)
+        else:
+            new_data = ChatMessage.make_image_message(w, image_context.values(), **kwargs)
         self.data.append(new_data)
 
     def addAssistantText(self, w: str, **kwargs):
@@ -45,7 +52,7 @@ def extendAssistantText(self, w: str) -> None:
                 if msg.content is None:
                     # this case is too weird, we just skip empty content
                     continue
-                elif type(msg) == str:
+                elif type(msg.content) == str:
                     # easy case
                     msg.content += w
                     return

ghostbox/Story.py

@@ -1 +1 @@
-0.21.2
+0.21.3

ghostbox/__VERSION__

@@ -55,7 +55,7 @@ def makeTaggedParser(default_params) -> TaggedArgumentParser:
                         tag=mktag(type=AT.Porcelain, group=AG.Backend))
     parser.add_argument("--backend", type=str, default=LLMBackend.generic.name, help="Backend to use. The default is `generic`, which conforms to the OpenAI REST API, and is supported by most LLM providers. Choosing a more specific backend may provide additional functionality. Other possible values are " + ", ".join([e.name for e in LLMBackend]) + ".",
                         tag=mktag(type=AT.Porcelain, group=AG.Backend, very_important=True))
-    parser.add_argument("--openai_api_key", type=str, default="", help="API key for OpenAI. Without the `--backend openai` option, this has no effect.",
+    parser.add_argument("--api_key", type=str, default="", help="API key for OpenAI. Without the `--backend openai` option, this has no effect.",
                         tag=mktag(type=AT.Plumbing, group=AG.OpenAI))    
     parser.add_argument("--max_length", type=int, default=300, help="Number of tokens to request from backend for generation. Generation is stopped when this number is exceeded. Negative values mean generation is unlimited and will terminate when the backend generates a stop token.",
                         tag=mktag(type=AT.Porcelain, group=AG.Generation, very_important=True))

ghostbox/_argparse.py

@@ -394,8 +394,6 @@ def generate(self, payload):
 
         if self._config["llamacpp_use_chat_completion_endpoint"]:
             endpoint_suffix = "/chat/completions"
-            # /chat/completions expects a more OAI like payload
-            llama_payload |= OpenAIBackend.dataFromPayload(llama_payload)
         else:
             endpoint_suffix = "/completion"
             if "tools" in payload:
@@ -446,8 +444,7 @@ def one_line_lambdas_for_python(r):
 
         if self._config["llamacpp_use_chat_completion_endpoint"]:
             endpoint_suffix = "/chat/completions"
-            # /chat/completions expects a more OAI like payload
-            llama_payload |= OpenAIBackend.dataFromPayload(llama_payload)
+
             final_callback = OpenAIBackend.makeOpenAICallback(
                 callback, last_result_callback=one_line_lambdas_for_python
             )
@@ -658,8 +655,6 @@ def generate(self, payload):
             "Content-Type": "application/json",
         }
         data = payload | {"max_tokens": payload["max_length"], "stream": False}
-        # the /V1/chat/completions endpoint expects structured data of user/assistant pairs
-        data |= self.dataFromPayload(payload)
 
         if "tools" in data:
             # see the llamacpp generate method fixme
@@ -720,8 +715,6 @@ def generateStreaming(self, payload, callback=lambda w: print(w)):
         }
 
         data = payload | {"stream": True, "stream_options": {"include_usage": True}}
-        # the /V1/chat/completions endpoint expects structured data of user/assistant pairs
-        data |= self.dataFromPayload(payload)
         self._last_request = data
 
         def one_line_lambdas_for_python(r):
@@ -744,47 +737,6 @@ def one_line_lambdas_for_python(r):
             return True
         return False
 
-    @staticmethod
-    def dataFromPayload(payload: Dict[str, Any]) -> Dict[str, Any]:
-        """Take a payload dictionary from Plumbing and return dictionary with elements specific to the chat/completions endpoint.
-        This expects payload to include the messages key, with various dictionaries in it, unlike other backends.
-        """
-        messages = [{"role": "system", "content": payload["system"]}]
-        # story is list of dicts with role and content keys
-        # we go through story one by one, mostly because of images
-        for story_item in payload["story"]:
-            if "image_id" in story_item:
-                # images is more complicated, see https://platform.openai.com/docs/guides/vision
-                # API wants the content field of an image message to be a list of dicts, not a string
-                # the dicts have the type field, which determines wether its a user msg (text) or image (image-url)
-                image_id = story_item["image_id"]
-                image_content_list = []
-                image_content_list.append(
-                    {"type": "text", "content": story_item["content"]}
-                )
-                if "images" not in payload or image_id not in payload["images"]:
-                    printerr("warning: image with id " + str(image_id) + " not found.")
-                    continue
-
-                # actually packaging the image
-                image_data = payload["images"][image_id]
-                ext = getImageExtension(image_data["url"], default="png")
-                base64_image = image_data["data"].decode("utf-8")
-                image_content_list.append(
-                    {
-                        "type": "image_url",
-                        "image_url": {"url": f"data:image/{ext};base64,{base64_image}"},
-                    }
-                )
-
-                messages.append(
-                    {"role": story_item["role"], "content": image_content_list}
-                )
-            else:
-                messages.append(story_item)
-
-        return {"messages": messages}
-
     def tokenize(self, w):
         headers = {
             "Authorization": f"Bearer {self.api_key}",