final tweaks

client/js/src/client.ts

@@ -437,7 +437,13 @@ export class Client {
 
 	private async connect_ws(url: string): Promise<void> {
 		return new Promise((resolve, reject) => {
-			const ws = new WebSocket(url);
+			let ws;
+			try {
+				ws = new WebSocket(url);
+			} catch (e) {
+				this.ws_map[url] = "failed";
+				return;
+			}
 
 			ws.onopen = () => {
 				resolve();

demo/streaming_filter/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: streaming_filter"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio opencv-python numpy "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import numpy as np\n", "import cv2\n", "\n", "def transform_cv2(frame, transform):\n", "    if transform == \"cartoon\":\n", "        # prepare color\n", "        img_color = cv2.pyrDown(cv2.pyrDown(frame))\n", "        for _ in range(6):\n", "            img_color = cv2.bilateralFilter(img_color, 9, 9, 7)\n", "        img_color = cv2.pyrUp(cv2.pyrUp(img_color))\n", "\n", "        # prepare edges\n", "        img_edges = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n", "        img_edges = cv2.adaptiveThreshold(\n", "            cv2.medianBlur(img_edges, 7),\n", "            255,\n", "            cv2.ADAPTIVE_THRESH_MEAN_C,\n", "            cv2.THRESH_BINARY,\n", "            9,\n", "            2,\n", "        )\n", "        img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)\n", "        # combine color and edges\n", "        img = cv2.bitwise_and(img_color, img_edges)\n", "        return img\n", "    elif transform == \"edges\":\n", "        # perform edge detection\n", "        img = cv2.cvtColor(cv2.Canny(frame, 100, 200), cv2.COLOR_GRAY2BGR)\n", "        return img\n", "    else:\n", "        return np.flipud(frame)\n", "\n", "with gr.Blocks() as demo:\n", "    with gr.Row():\n", "        with gr.Column():\n", "            transform = gr.Dropdown(choices=[\"cartoon\", \"edges\", \"flip\"],\n", "                                    value=\"flip\", label=\"Transformation\")\n", "            input_img = gr.Image(sources=[\"webcam\"], type=\"numpy\")\n", "        with gr.Column():\n", "            output_img = gr.Image(format=\"base64\")\n", "        dep = input_img.stream(transform_cv2, [input_img, transform], [output_img],\n", "                                time_limit=30, stream_every=0.1, concurrency_limit=30)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: streaming_filter"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio opencv-python numpy "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import numpy as np\n", "import cv2\n", "\n", "def transform_cv2(frame, transform):\n", "    if transform == \"cartoon\":\n", "        # prepare color\n", "        img_color = cv2.pyrDown(cv2.pyrDown(frame))\n", "        for _ in range(6):\n", "            img_color = cv2.bilateralFilter(img_color, 9, 9, 7)\n", "        img_color = cv2.pyrUp(cv2.pyrUp(img_color))\n", "\n", "        # prepare edges\n", "        img_edges = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n", "        img_edges = cv2.adaptiveThreshold(\n", "            cv2.medianBlur(img_edges, 7),\n", "            255,\n", "            cv2.ADAPTIVE_THRESH_MEAN_C,\n", "            cv2.THRESH_BINARY,\n", "            9,\n", "            2,\n", "        )\n", "        img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)\n", "        # combine color and edges\n", "        img = cv2.bitwise_and(img_color, img_edges)\n", "        return img\n", "    elif transform == \"edges\":\n", "        # perform edge detection\n", "        img = cv2.cvtColor(cv2.Canny(frame, 100, 200), cv2.COLOR_GRAY2BGR)\n", "        return img\n", "    else:\n", "        return np.flipud(frame)\n", "\n", "with gr.Blocks() as demo:\n", "    with gr.Row():\n", "        with gr.Column():\n", "            transform = gr.Dropdown(choices=[\"cartoon\", \"edges\", \"flip\"],\n", "                                    value=\"flip\", label=\"Transformation\")\n", "            input_img = gr.Image(sources=[\"webcam\"], type=\"numpy\")\n", "        with gr.Column():\n", "            output_img = gr.Image(streaming=True)\n", "        dep = input_img.stream(transform_cv2, [input_img, transform], [output_img],\n", "                                time_limit=30, stream_every=0.1, concurrency_limit=30)\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/streaming_filter/run.py

@@ -38,7 +38,7 @@ def transform_cv2(frame, transform):
                                     value="flip", label="Transformation")
             input_img = gr.Image(sources=["webcam"], type="numpy")
         with gr.Column():
-            output_img = gr.Image(format="base64")
+            output_img = gr.Image(streaming=True)
         dep = input_img.stream(transform_cv2, [input_img, transform], [output_img],
                                 time_limit=30, stream_every=0.1, concurrency_limit=30)
 

demo/streaming_filter_unified/run.ipynb

@@ -1 +1 @@
-{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: streaming_filter_unified"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio opencv-python numpy "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import numpy as np\n", "import cv2\n", "\n", "def transform_cv2(frame, transform):\n", "    if transform == \"cartoon\":\n", "        # prepare color\n", "        img_color = cv2.pyrDown(cv2.pyrDown(frame))\n", "        for _ in range(6):\n", "            img_color = cv2.bilateralFilter(img_color, 9, 9, 7)\n", "        img_color = cv2.pyrUp(cv2.pyrUp(img_color))\n", "\n", "        # prepare edges\n", "        img_edges = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n", "        img_edges = cv2.adaptiveThreshold(\n", "            cv2.medianBlur(img_edges, 7),\n", "            255,\n", "            cv2.ADAPTIVE_THRESH_MEAN_C,\n", "            cv2.THRESH_BINARY,\n", "            9,\n", "            2,\n", "        )\n", "        img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)\n", "        # combine color and edges\n", "        img = cv2.bitwise_and(img_color, img_edges)\n", "        return img\n", "    elif transform == \"edges\":\n", "        # perform edge detection\n", "        img = cv2.cvtColor(cv2.Canny(frame, 100, 200), cv2.COLOR_GRAY2BGR)\n", "        return img\n", "    else:\n", "        return np.flipud(frame)\n", "\n", "\n", "css=\"\"\".my-group {max-width: 500px !important; max-height: 500px !important;}\n", "            .my-column {display: flex !important; justify-content: center !important; align-items: center !important};\"\"\"\n", "\n", "with gr.Blocks(css=css) as demo:\n", "    with gr.Column(elem_classes=[\"my-column\"]):\n", "        with gr.Group(elem_classes=[\"my-group\"]):\n", "            transform = gr.Dropdown(choices=[\"cartoon\", \"edges\", \"flip\"],\n", "                                    value=\"flip\", label=\"Transformation\")\n", "            input_img = gr.Image(sources=[\"webcam\"], type=\"numpy\", format=\"base64\")\n", "    input_img.stream(transform_cv2, [input_img, transform], [input_img], time_limit=30, stream_every=0.1)\n", "\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}
+{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: streaming_filter_unified"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio opencv-python numpy "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import numpy as np\n", "import cv2\n", "\n", "def transform_cv2(frame, transform):\n", "    if transform == \"cartoon\":\n", "        # prepare color\n", "        img_color = cv2.pyrDown(cv2.pyrDown(frame))\n", "        for _ in range(6):\n", "            img_color = cv2.bilateralFilter(img_color, 9, 9, 7)\n", "        img_color = cv2.pyrUp(cv2.pyrUp(img_color))\n", "\n", "        # prepare edges\n", "        img_edges = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n", "        img_edges = cv2.adaptiveThreshold(\n", "            cv2.medianBlur(img_edges, 7),\n", "            255,\n", "            cv2.ADAPTIVE_THRESH_MEAN_C,\n", "            cv2.THRESH_BINARY,\n", "            9,\n", "            2,\n", "        )\n", "        img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)\n", "        # combine color and edges\n", "        img = cv2.bitwise_and(img_color, img_edges)\n", "        return img\n", "    elif transform == \"edges\":\n", "        # perform edge detection\n", "        img = cv2.cvtColor(cv2.Canny(frame, 100, 200), cv2.COLOR_GRAY2BGR)\n", "        return img\n", "    else:\n", "        return np.flipud(frame)\n", "\n", "\n", "css=\"\"\".my-group {max-width: 500px !important; max-height: 500px !important;}\n", "            .my-column {display: flex !important; justify-content: center !important; align-items: center !important};\"\"\"\n", "\n", "with gr.Blocks(css=css) as demo:\n", "    with gr.Column(elem_classes=[\"my-column\"]):\n", "        with gr.Group(elem_classes=[\"my-group\"]):\n", "            transform = gr.Dropdown(choices=[\"cartoon\", \"edges\", \"flip\"],\n", "                                    value=\"flip\", label=\"Transformation\")\n", "            input_img = gr.Image(sources=[\"webcam\"], type=\"numpy\", streaming=True)\n", "    input_img.stream(transform_cv2, [input_img, transform], [input_img], time_limit=30, stream_every=0.1)\n", "\n", "\n", "if __name__ == \"__main__\":\n", "    demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5}

demo/streaming_filter_unified/run.py

@@ -40,7 +40,7 @@ def transform_cv2(frame, transform):
         with gr.Group(elem_classes=["my-group"]):
             transform = gr.Dropdown(choices=["cartoon", "edges", "flip"],
                                     value="flip", label="Transformation")
-            input_img = gr.Image(sources=["webcam"], type="numpy", format="base64")
+            input_img = gr.Image(sources=["webcam"], type="numpy", streaming=True)
     input_img.stream(transform_cv2, [input_img, transform], [input_img], time_limit=30, stream_every=0.1)
 
 

gradio/components/image.py

@@ -106,7 +106,7 @@ def __init__(
         """
         Parameters:
             value: A PIL Image, numpy array, path or URL for the default value that Image component is going to take. If callable, the function will be called whenever the app loads to set the initial value of the component.
-            format: File format (e.g. "png" or "gif") or "base64". Used to save image if it does not already have a valid format (e.g. if the image is being returned to the frontend as a numpy array or PIL Image). If not "base64", the format should be supported by the PIL library. Applies both when this component is used as an input or output. This parameter has no effect on SVG files.
+            format: File format (e.g. "png" or "gif"). Used to save image if it does not already have a valid format (e.g. if the image is being returned to the frontend as a numpy array or PIL Image). The format should be supported by the PIL library. Applies both when this component is used as an input or output. This parameter has no effect on SVG files.
             height: The height of the component, specified in pixels if a number is passed, or in CSS units if a string is passed. This has no effect on the preprocessed image file or numpy array, but will affect the displayed image.
             width: The width of the component, specified in pixels if a number is passed, or in CSS units if a string is passed. This has no effect on the preprocessed image file or numpy array, but will affect the displayed image.
             image_mode: The pixel format and color depth that the image should be loaded and preprocessed as. "RGB" will load the image as a color image, or "L" as black-and-white. See https://pillow.readthedocs.io/en/stable/handbook/concepts.html for other supported image modes and their meaning. This parameter has no effect on SVG or GIF files. If set to None, the image_mode will be inferred from the image file type (e.g. "RGBA" for a .png image, "RGB" in most other cases).
@@ -122,7 +122,7 @@ def __init__(
             min_width: minimum pixel width, will wrap if not sufficient screen space to satisfy this value. If a certain scale value results in this Component being narrower than min_width, the min_width parameter will be respected first.
             interactive: if True, will allow users to upload and edit an image; if False, can only be used to display images. If not provided, this is inferred based on whether the component is used as an input or output.
             visible: If False, component will be hidden.
-            streaming: If True when used in a `live` interface, will automatically stream webcam feed. Only valid is source is 'webcam'.
+            streaming: If True when used in a `live` interface, will automatically stream webcam feed. Only valid is source is 'webcam'. If the component is an output component, will automatically convert images to base64.
             elem_id: An optional string that is assigned as the id of this component in the HTML DOM. Can be used for targeting CSS styles.
             elem_classes: An optional list of strings that are assigned as the classes of this component in the HTML DOM. Can be used for targeting CSS styles.
             render: If False, component will not render be rendered in the Blocks context. Should be used if the intention is to assign event listeners now but render the component later.
@@ -264,7 +264,7 @@ def postprocess(
             return None
         if isinstance(value, str) and value.lower().endswith(".svg"):
             return ImageData(path=value, orig_name=Path(value).name)
-        if self.format == "base64":
+        if self.streaming:
             if isinstance(value, np.ndarray):
                 return Base64ImageData(
                     url=image_utils.encode_image_array_to_base64(value)
@@ -281,7 +281,7 @@ def postprocess(
         return ImageData(path=saved, orig_name=orig_name)
 
     def api_info_as_output(self) -> dict[str, Any]:
-        if self.format == "base64":
+        if self.streaming == "base64":
             schema = Base64ImageData.model_json_schema()
             schema.pop("description", None)
             return schema

guides/04_additional-features/03_streaming-inputs.md

@@ -25,7 +25,7 @@ $demo_streaming_filter
 
 You will notice that if you change the filter value it will immediately take effect in the output stream. That is an important difference of stream events in comparison to other Gradio events. The input values of the stream can be changed while the stream is being processed. 
 
-Tip: We set the "format" parameter of the image output component to be "base64". That way, the server does not have to spend time saving the image to a file. This will help reduce latency.
+Tip: We set the "streaming" parameter of the image output component to be "True". Doing so lets the server automatically convert our output images into base64 format, a format that is efficient for streaming.
 
 ## Unified Image Demos