diff --git a/client/js/src/client.ts b/client/js/src/client.ts index 3febc2815a954..47058adb75522 100644 --- a/client/js/src/client.ts +++ b/client/js/src/client.ts @@ -437,7 +437,13 @@ export class Client { private async connect_ws(url: string): Promise { return new Promise((resolve, reject) => { - const ws = new WebSocket(url); + let ws; + try { + ws = new WebSocket(url); + } catch (e) { + this.ws_map[url] = "failed"; + return; + } ws.onopen = () => { resolve(); diff --git a/demo/streaming_filter/run.ipynb b/demo/streaming_filter/run.ipynb index 8babf7be4ce75..161dde8a85988 100644 --- a/demo/streaming_filter/run.ipynb +++ b/demo/streaming_filter/run.ipynb @@ -1 +1 @@ -{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: streaming_filter"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio opencv-python numpy "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import numpy as np\n", "import cv2\n", "\n", "def transform_cv2(frame, transform):\n", " if transform == \"cartoon\":\n", " # prepare color\n", " img_color = cv2.pyrDown(cv2.pyrDown(frame))\n", " for _ in range(6):\n", " img_color = cv2.bilateralFilter(img_color, 9, 9, 7)\n", " img_color = cv2.pyrUp(cv2.pyrUp(img_color))\n", "\n", " # prepare edges\n", " img_edges = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n", " img_edges = cv2.adaptiveThreshold(\n", " cv2.medianBlur(img_edges, 7),\n", " 255,\n", " cv2.ADAPTIVE_THRESH_MEAN_C,\n", " cv2.THRESH_BINARY,\n", " 9,\n", " 2,\n", " )\n", " img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)\n", " # combine color and edges\n", " img = cv2.bitwise_and(img_color, img_edges)\n", " return img\n", " elif transform == \"edges\":\n", " # perform edge detection\n", " img = cv2.cvtColor(cv2.Canny(frame, 100, 200), cv2.COLOR_GRAY2BGR)\n", " return img\n", " else:\n", " return np.flipud(frame)\n", "\n", "with gr.Blocks() as demo:\n", " with gr.Row():\n", " with gr.Column():\n", " transform = gr.Dropdown(choices=[\"cartoon\", \"edges\", \"flip\"],\n", " value=\"flip\", label=\"Transformation\")\n", " input_img = gr.Image(sources=[\"webcam\"], type=\"numpy\")\n", " with gr.Column():\n", " output_img = gr.Image(format=\"base64\")\n", " dep = input_img.stream(transform_cv2, [input_img, transform], [output_img],\n", " time_limit=30, stream_every=0.1, concurrency_limit=30)\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5} \ No newline at end of file +{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: streaming_filter"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio opencv-python numpy "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import numpy as np\n", "import cv2\n", "\n", "def transform_cv2(frame, transform):\n", " if transform == \"cartoon\":\n", " # prepare color\n", " img_color = cv2.pyrDown(cv2.pyrDown(frame))\n", " for _ in range(6):\n", " img_color = cv2.bilateralFilter(img_color, 9, 9, 7)\n", " img_color = cv2.pyrUp(cv2.pyrUp(img_color))\n", "\n", " # prepare edges\n", " img_edges = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n", " img_edges = cv2.adaptiveThreshold(\n", " cv2.medianBlur(img_edges, 7),\n", " 255,\n", " cv2.ADAPTIVE_THRESH_MEAN_C,\n", " cv2.THRESH_BINARY,\n", " 9,\n", " 2,\n", " )\n", " img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)\n", " # combine color and edges\n", " img = cv2.bitwise_and(img_color, img_edges)\n", " return img\n", " elif transform == \"edges\":\n", " # perform edge detection\n", " img = cv2.cvtColor(cv2.Canny(frame, 100, 200), cv2.COLOR_GRAY2BGR)\n", " return img\n", " else:\n", " return np.flipud(frame)\n", "\n", "with gr.Blocks() as demo:\n", " with gr.Row():\n", " with gr.Column():\n", " transform = gr.Dropdown(choices=[\"cartoon\", \"edges\", \"flip\"],\n", " value=\"flip\", label=\"Transformation\")\n", " input_img = gr.Image(sources=[\"webcam\"], type=\"numpy\")\n", " with gr.Column():\n", " output_img = gr.Image(streaming=True)\n", " dep = input_img.stream(transform_cv2, [input_img, transform], [output_img],\n", " time_limit=30, stream_every=0.1, concurrency_limit=30)\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5} \ No newline at end of file diff --git a/demo/streaming_filter/run.py b/demo/streaming_filter/run.py index 9f5535f0c65f2..dca23e5602ac3 100644 --- a/demo/streaming_filter/run.py +++ b/demo/streaming_filter/run.py @@ -38,7 +38,7 @@ def transform_cv2(frame, transform): value="flip", label="Transformation") input_img = gr.Image(sources=["webcam"], type="numpy") with gr.Column(): - output_img = gr.Image(format="base64") + output_img = gr.Image(streaming=True) dep = input_img.stream(transform_cv2, [input_img, transform], [output_img], time_limit=30, stream_every=0.1, concurrency_limit=30) diff --git a/demo/streaming_filter_unified/run.ipynb b/demo/streaming_filter_unified/run.ipynb index a56bec5776e67..bbb5dfcc4983a 100644 --- a/demo/streaming_filter_unified/run.ipynb +++ b/demo/streaming_filter_unified/run.ipynb @@ -1 +1 @@ -{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: streaming_filter_unified"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio opencv-python numpy "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import numpy as np\n", "import cv2\n", "\n", "def transform_cv2(frame, transform):\n", " if transform == \"cartoon\":\n", " # prepare color\n", " img_color = cv2.pyrDown(cv2.pyrDown(frame))\n", " for _ in range(6):\n", " img_color = cv2.bilateralFilter(img_color, 9, 9, 7)\n", " img_color = cv2.pyrUp(cv2.pyrUp(img_color))\n", "\n", " # prepare edges\n", " img_edges = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n", " img_edges = cv2.adaptiveThreshold(\n", " cv2.medianBlur(img_edges, 7),\n", " 255,\n", " cv2.ADAPTIVE_THRESH_MEAN_C,\n", " cv2.THRESH_BINARY,\n", " 9,\n", " 2,\n", " )\n", " img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)\n", " # combine color and edges\n", " img = cv2.bitwise_and(img_color, img_edges)\n", " return img\n", " elif transform == \"edges\":\n", " # perform edge detection\n", " img = cv2.cvtColor(cv2.Canny(frame, 100, 200), cv2.COLOR_GRAY2BGR)\n", " return img\n", " else:\n", " return np.flipud(frame)\n", "\n", "\n", "css=\"\"\".my-group {max-width: 500px !important; max-height: 500px !important;}\n", " .my-column {display: flex !important; justify-content: center !important; align-items: center !important};\"\"\"\n", "\n", "with gr.Blocks(css=css) as demo:\n", " with gr.Column(elem_classes=[\"my-column\"]):\n", " with gr.Group(elem_classes=[\"my-group\"]):\n", " transform = gr.Dropdown(choices=[\"cartoon\", \"edges\", \"flip\"],\n", " value=\"flip\", label=\"Transformation\")\n", " input_img = gr.Image(sources=[\"webcam\"], type=\"numpy\", format=\"base64\")\n", " input_img.stream(transform_cv2, [input_img, transform], [input_img], time_limit=30, stream_every=0.1)\n", "\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5} \ No newline at end of file +{"cells": [{"cell_type": "markdown", "id": "302934307671667531413257853548643485645", "metadata": {}, "source": ["# Gradio Demo: streaming_filter_unified"]}, {"cell_type": "code", "execution_count": null, "id": "272996653310673477252411125948039410165", "metadata": {}, "outputs": [], "source": ["!pip install -q gradio opencv-python numpy "]}, {"cell_type": "code", "execution_count": null, "id": "288918539441861185822528903084949547379", "metadata": {}, "outputs": [], "source": ["import gradio as gr\n", "import numpy as np\n", "import cv2\n", "\n", "def transform_cv2(frame, transform):\n", " if transform == \"cartoon\":\n", " # prepare color\n", " img_color = cv2.pyrDown(cv2.pyrDown(frame))\n", " for _ in range(6):\n", " img_color = cv2.bilateralFilter(img_color, 9, 9, 7)\n", " img_color = cv2.pyrUp(cv2.pyrUp(img_color))\n", "\n", " # prepare edges\n", " img_edges = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n", " img_edges = cv2.adaptiveThreshold(\n", " cv2.medianBlur(img_edges, 7),\n", " 255,\n", " cv2.ADAPTIVE_THRESH_MEAN_C,\n", " cv2.THRESH_BINARY,\n", " 9,\n", " 2,\n", " )\n", " img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)\n", " # combine color and edges\n", " img = cv2.bitwise_and(img_color, img_edges)\n", " return img\n", " elif transform == \"edges\":\n", " # perform edge detection\n", " img = cv2.cvtColor(cv2.Canny(frame, 100, 200), cv2.COLOR_GRAY2BGR)\n", " return img\n", " else:\n", " return np.flipud(frame)\n", "\n", "\n", "css=\"\"\".my-group {max-width: 500px !important; max-height: 500px !important;}\n", " .my-column {display: flex !important; justify-content: center !important; align-items: center !important};\"\"\"\n", "\n", "with gr.Blocks(css=css) as demo:\n", " with gr.Column(elem_classes=[\"my-column\"]):\n", " with gr.Group(elem_classes=[\"my-group\"]):\n", " transform = gr.Dropdown(choices=[\"cartoon\", \"edges\", \"flip\"],\n", " value=\"flip\", label=\"Transformation\")\n", " input_img = gr.Image(sources=[\"webcam\"], type=\"numpy\", streaming=True)\n", " input_img.stream(transform_cv2, [input_img, transform], [input_img], time_limit=30, stream_every=0.1)\n", "\n", "\n", "if __name__ == \"__main__\":\n", " demo.launch()\n"]}], "metadata": {}, "nbformat": 4, "nbformat_minor": 5} \ No newline at end of file diff --git a/demo/streaming_filter_unified/run.py b/demo/streaming_filter_unified/run.py index 67e0732aca915..fd8ce7956484d 100644 --- a/demo/streaming_filter_unified/run.py +++ b/demo/streaming_filter_unified/run.py @@ -40,7 +40,7 @@ def transform_cv2(frame, transform): with gr.Group(elem_classes=["my-group"]): transform = gr.Dropdown(choices=["cartoon", "edges", "flip"], value="flip", label="Transformation") - input_img = gr.Image(sources=["webcam"], type="numpy", format="base64") + input_img = gr.Image(sources=["webcam"], type="numpy", streaming=True) input_img.stream(transform_cv2, [input_img, transform], [input_img], time_limit=30, stream_every=0.1) diff --git a/gradio/components/image.py b/gradio/components/image.py index 2f6e34a3ac750..2e2a9637bac17 100644 --- a/gradio/components/image.py +++ b/gradio/components/image.py @@ -106,7 +106,7 @@ def __init__( """ Parameters: value: A PIL Image, numpy array, path or URL for the default value that Image component is going to take. If callable, the function will be called whenever the app loads to set the initial value of the component. - format: File format (e.g. "png" or "gif") or "base64". Used to save image if it does not already have a valid format (e.g. if the image is being returned to the frontend as a numpy array or PIL Image). If not "base64", the format should be supported by the PIL library. Applies both when this component is used as an input or output. This parameter has no effect on SVG files. + format: File format (e.g. "png" or "gif"). Used to save image if it does not already have a valid format (e.g. if the image is being returned to the frontend as a numpy array or PIL Image). The format should be supported by the PIL library. Applies both when this component is used as an input or output. This parameter has no effect on SVG files. height: The height of the component, specified in pixels if a number is passed, or in CSS units if a string is passed. This has no effect on the preprocessed image file or numpy array, but will affect the displayed image. width: The width of the component, specified in pixels if a number is passed, or in CSS units if a string is passed. This has no effect on the preprocessed image file or numpy array, but will affect the displayed image. image_mode: The pixel format and color depth that the image should be loaded and preprocessed as. "RGB" will load the image as a color image, or "L" as black-and-white. See https://pillow.readthedocs.io/en/stable/handbook/concepts.html for other supported image modes and their meaning. This parameter has no effect on SVG or GIF files. If set to None, the image_mode will be inferred from the image file type (e.g. "RGBA" for a .png image, "RGB" in most other cases). @@ -122,7 +122,7 @@ def __init__( min_width: minimum pixel width, will wrap if not sufficient screen space to satisfy this value. If a certain scale value results in this Component being narrower than min_width, the min_width parameter will be respected first. interactive: if True, will allow users to upload and edit an image; if False, can only be used to display images. If not provided, this is inferred based on whether the component is used as an input or output. visible: If False, component will be hidden. - streaming: If True when used in a `live` interface, will automatically stream webcam feed. Only valid is source is 'webcam'. + streaming: If True when used in a `live` interface, will automatically stream webcam feed. Only valid is source is 'webcam'. If the component is an output component, will automatically convert images to base64. elem_id: An optional string that is assigned as the id of this component in the HTML DOM. Can be used for targeting CSS styles. elem_classes: An optional list of strings that are assigned as the classes of this component in the HTML DOM. Can be used for targeting CSS styles. render: If False, component will not render be rendered in the Blocks context. Should be used if the intention is to assign event listeners now but render the component later. @@ -264,7 +264,7 @@ def postprocess( return None if isinstance(value, str) and value.lower().endswith(".svg"): return ImageData(path=value, orig_name=Path(value).name) - if self.format == "base64": + if self.streaming: if isinstance(value, np.ndarray): return Base64ImageData( url=image_utils.encode_image_array_to_base64(value) @@ -281,7 +281,7 @@ def postprocess( return ImageData(path=saved, orig_name=orig_name) def api_info_as_output(self) -> dict[str, Any]: - if self.format == "base64": + if self.streaming == "base64": schema = Base64ImageData.model_json_schema() schema.pop("description", None) return schema diff --git a/guides/04_additional-features/03_streaming-inputs.md b/guides/04_additional-features/03_streaming-inputs.md index f324b2e13fd60..60729ec47ca88 100644 --- a/guides/04_additional-features/03_streaming-inputs.md +++ b/guides/04_additional-features/03_streaming-inputs.md @@ -25,7 +25,7 @@ $demo_streaming_filter You will notice that if you change the filter value it will immediately take effect in the output stream. That is an important difference of stream events in comparison to other Gradio events. The input values of the stream can be changed while the stream is being processed. -Tip: We set the "format" parameter of the image output component to be "base64". That way, the server does not have to spend time saving the image to a file. This will help reduce latency. +Tip: We set the "streaming" parameter of the image output component to be "True". Doing so lets the server automatically convert our output images into base64 format, a format that is efficient for streaming. ## Unified Image Demos