webb_vs_hubble.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Carina Nebula (NGC 3324): Webb vs Hubble\n",
    "\n",
    "Compare Webb and Hubble images side-by-side, and also sonified \"side-by-side\" (i.e., panned to the left and right).\n",
    "\n",
    "Multiple sounds are generated from each image:\n",
    "\n",
    "- Musical notes triggered when hovering over stars. The note triggered depends on the pixel hue, from red (low note) to blue (high note)\n",
    "- Background noise level modulated by the local \"noise\" in the image.\n",
    "- Background drone modulated by the RGB image bands."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import ipycanvas\n",
    "import ipyevents\n",
    "import ipytone\n",
    "import ipywidgets\n",
    "import skimage\n",
    "\n",
    "from skimage import filters\n",
    "from skimage.morphology import disk, binary_dilation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and pre-process images\n",
    "\n",
    "The image files are not included in this repository. You can download it here:\n",
    "\n",
    "- Hubble: https://hubblesite.org/contents/media/images/2008/34/2405-Image.html\n",
    "- Webb: https://webbtelescope.org/contents/media/images/2022/031/01G77PKB8NKR7S8Z6HBXMYATGJ\n",
    "\n",
    "The Hubble image has been roughly aligned with the Webb image by calibrating `skimage.transform.SimilarityTransform()` on a set of ~100 control points defined by hand."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hubble, webb = skimage.io.ImageCollection([\"data/hubble_export.tif\", \"data/webb_export.tif\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#width = 560\n",
    "#height = 300\n",
    "width = 700\n",
    "height = 400\n",
    "\n",
    "hubble_r = skimage.transform.resize(hubble, (height, width), preserve_range=True)\n",
    "webb_r = skimage.transform.resize(webb, (height, width), preserve_range=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create binary masks that extract stars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def detect_stars(img, threshold=0.025):\n",
    "    \"\"\"Simple image filter (+ dilation) to extract stars as a\n",
    "    binary mask)\n",
    "    \n",
    "    \"\"\"\n",
    "    bw = skimage.color.rgb2gray(img / 256)\n",
    "    fltr = filters.rank.median(skimage.util.img_as_ubyte(bw), disk(1))\n",
    "    mask = bw - fltr / 256 > threshold\n",
    "    dilated_mask = binary_dilation(mask, disk(1))\n",
    "    return dilated_mask\n",
    "\n",
    "\n",
    "hubble_star_mask = detect_stars(hubble_r)\n",
    "webb_star_mask = detect_stars(webb_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create frequency arrays that determine the musical note to play when hovering on a star"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hue2freq(hue, freq_range=(100, 4000)):\n",
    "    \"\"\"convert hue given on interval [0, 1] to a frequency\n",
    "    \n",
    "    The returned frequency is within in the given range.\n",
    "    The frequency range has periodic boundaries.\n",
    "\n",
    "    \"\"\"\n",
    "    fmin = freq_range[0]\n",
    "    fmax = freq_range[1]\n",
    "    ampl = fmax - fmin\n",
    "    return fmin + ampl * np.sin(hue * np.pi)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hubble_hsv = skimage.color.rgb2hsv(hubble_r / 256)\n",
    "hubble_freq = hue2freq(hubble_hsv[..., 0])\n",
    "\n",
    "webb_hsv = skimage.color.rgb2hsv(webb_r / 256)\n",
    "webb_freq = hue2freq(webb_hsv[..., 0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create arrays that determine the level of noise (filter images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def detect_noise(img):\n",
    "    \"\"\"Simple filter to detect the local amount of \"noise\"\n",
    "    in the image.\n",
    "    \n",
    "    \"\"\"\n",
    "    bw = skimage.color.rgb2gray(img) / 256\n",
    "    bw_sq = bw * bw\n",
    "\n",
    "    region = disk(3)\n",
    "    mean_bw = filters.rank.mean(\n",
    "        skimage.util.img_as_ubyte(bw), footprint=region\n",
    "    ).astype(np.float32)\n",
    "    mean_bw_sq = filters.rank.mean(\n",
    "        skimage.util.img_as_ubyte(bw_sq), footprint=region\n",
    "    ).astype(np.float32)\n",
    "\n",
    "    sq_mean_bw = mean_bw * mean_bw\n",
    "    std = np.sqrt(sq_mean_bw - mean_bw_sq)\n",
    "    noise = std / (std.max() * 2)\n",
    "    \n",
    "    return noise\n",
    "\n",
    "\n",
    "hubble_noise = detect_noise(hubble_r)\n",
    "webb_noise = detect_noise(webb_r)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup canvas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_height = 60\n",
    "\n",
    "\n",
    "def create_mcanvas(img, title):\n",
    "    mcanvas = ipycanvas.MultiCanvas(3, width=width, height=height + title_height)\n",
    "    \n",
    "    background = mcanvas[0]\n",
    "    background.fill_style = \"black\"\n",
    "    background.fill_rect(0, 0, width, height + title_height)\n",
    "    background.put_image_data(img, x=0, y=title_height)\n",
    "    \n",
    "    background.stroke_style = \"white\"\n",
    "    background.fill_style = \"white\"\n",
    "    background.font = f\"{title_height - 40}px sans-serif\"\n",
    "    background.text_baseline = \"top\"\n",
    "    background.fill_text(title, 20, 20)\n",
    "    \n",
    "    return mcanvas\n",
    "    \n",
    "\n",
    "hubble_canvas = create_mcanvas(hubble_r, \"HUBBLE [L]\")\n",
    "webb_canvas = create_mcanvas(webb_r, \"WEBB [R]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_circle(canvas, fill_color):\n",
    "    canvas.stroke_style = \"white\"\n",
    "    canvas.fill_style = fill_color\n",
    "    canvas.line_width = 2.0\n",
    "    \n",
    "    xc = width - 40\n",
    "    yc = title_height // 2\n",
    "    canvas.fill_circle(xc, yc, 10)\n",
    "    canvas.stroke_circle(xc, yc, 10)\n",
    "    \n",
    "\n",
    "draw_circle(hubble_canvas[2], \"black\")\n",
    "draw_circle(webb_canvas[2], \"black\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup sounds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multiband_img_sound_setup(pan=0, detune=0):\n",
    "    reverb = ipytone.Reverb().to_destination()\n",
    "    panner = ipytone.Panner(pan=pan).connect(reverb)\n",
    "    \n",
    "    reverb.wet.value = 0.6\n",
    "    \n",
    "    noise_gain = ipytone.Gain(gain=0)\n",
    "    noise_filter = ipytone.Filter(type=\"lowpass\", frequency=1500)\n",
    "    noise = ipytone.Noise(type=\"brown\").chain(noise_gain, noise_filter, panner).start()\n",
    "    \n",
    "    gain_r = ipytone.Gain(gain=0)\n",
    "    osc_r = ipytone.FatOscillator(frequency=\"A2\").chain(gain_r, panner).start()\n",
    "    osc_r.type = \"sine\"\n",
    "    osc_r.detune.value = detune\n",
    "    gain_g = ipytone.Gain(gain=0)\n",
    "    osc_g = ipytone.FatOscillator(frequency=\"A4\").chain(gain_g, panner).start()\n",
    "    osc_g.type = \"triangle\"\n",
    "    osc_g.detune.value = detune\n",
    "    gain_b = ipytone.Gain(gain=0)\n",
    "    osc_b = ipytone.FatOscillator(frequency=\"C6\").chain(gain_b, panner).start()\n",
    "    osc_b.type = \"sine\"\n",
    "    osc_b.detune.value = detune\n",
    "        \n",
    "    psynth = ipytone.PolySynth(volume=-4, max_polyphony=10).connect(panner)\n",
    "    psynth.voice.oscillator.type = \"amsine\"\n",
    "    psynth.voice.envelope.release = 1.8\n",
    "    psynth.voice.envelope.attack = 0.25\n",
    "    \n",
    "    return {\n",
    "        \"noise_gain\": noise_gain,\n",
    "        \"noise_filter\": noise_filter,\n",
    "        \"noise\": noise,\n",
    "        \"gain_r\": gain_r,\n",
    "        \"osc_r\": osc_r,\n",
    "        \"gain_g\": gain_g,\n",
    "        \"osc_g\": osc_g,\n",
    "        \"gain_b\": gain_b,\n",
    "        \"osc_b\": osc_b,\n",
    "        \"psynth\": psynth,\n",
    "        \"panner\": panner,\n",
    "        \"reverb\": reverb,\n",
    "    }\n",
    "\n",
    "\n",
    "def dispose(sound_setup):\n",
    "    for widget in sound_setup.values():\n",
    "        widget.dispose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hubble_sound = multiband_img_sound_setup(pan=-1)\n",
    "webb_sound = multiband_img_sound_setup(pan=1, detune=40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup mouse event handlers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hubble_canvas[1].stroke_style = \"white\"\n",
    "hubble_canvas[1].line_width = 2.0\n",
    "webb_canvas[1].stroke_style = \"white\"\n",
    "webb_canvas[1].line_width = 2.0\n",
    "\n",
    "\n",
    "def draw_pointer(x, y):\n",
    "    for mcv in [hubble_canvas, webb_canvas]:\n",
    "        mcv[1].clear()\n",
    "        mcv[1].stroke_circle(x, y + title_height, 15)\n",
    "\n",
    "\n",
    "def update_circle_color(x ,y):\n",
    "    t = [\n",
    "        (hubble_r, hubble_star_mask, hubble_canvas),\n",
    "        (webb_r, webb_star_mask, webb_canvas),\n",
    "    ]\n",
    "    \n",
    "    for img, mask, mcv in t:\n",
    "        if mask[y, x]:\n",
    "            r, g, b = img[y, x].astype(np.uint8)\n",
    "            color = f\"rgb({r}, {g}, {b})\"\n",
    "        else:\n",
    "            color = \"black\"\n",
    "    \n",
    "        draw_circle(mcv[2], color)\n",
    "\n",
    "\n",
    "def trigger_note(x, y, mask, freq, hsv, sound):\n",
    "    if mask[y, x]:\n",
    "        note = freq[y, x]\n",
    "        vel = hsv[y, x, 2]\n",
    "        sound[\"psynth\"].trigger_attack_release(note, 0.1, velocity=vel)\n",
    "\n",
    "\n",
    "def adjust_gains(x, y, img, sound):\n",
    "    r, g, b = img[y, x] / 256\n",
    "    sound[\"gain_r\"].gain.ramp_to(r, 0.2)\n",
    "    sound[\"gain_g\"].gain.ramp_to(g / 3, 0.2)\n",
    "    sound[\"gain_b\"].gain.ramp_to(b / 3, 0.2)\n",
    "\n",
    "\n",
    "def update_sound(x, y):\n",
    "    trigger_note(x, y, hubble_star_mask, hubble_freq, hubble_hsv, hubble_sound)\n",
    "    trigger_note(x, y, webb_star_mask, webb_freq, webb_hsv, webb_sound)\n",
    "    \n",
    "    hubble_sound[\"noise_gain\"].gain.ramp_to(hubble_noise[y, x], 0.1)\n",
    "    webb_sound[\"noise_gain\"].gain.ramp_to(webb_noise[y, x], 0.1)\n",
    "    \n",
    "    adjust_gains(x, y, hubble_r, hubble_sound)\n",
    "    adjust_gains(x, y, webb_r, webb_sound)\n",
    "\n",
    "\n",
    "def action(event):\n",
    "    x = int(event[\"relativeX\"])\n",
    "    y = int(event[\"relativeY\"] - title_height)\n",
    "    \n",
    "    if y < 0:\n",
    "        return\n",
    "    \n",
    "    draw_pointer(x, y)\n",
    "    update_circle_color(x, y)\n",
    "    update_sound(x, y)\n",
    "    \n",
    "\n",
    "event = ipyevents.Event(source=hubble_canvas, watched_events=[\"mousemove\"], wait=120)\n",
    "event.on_dom_event(action)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's play"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ipywidgets.HBox([hubble_canvas, webb_canvas])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean-up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dispose(hubble_sound)\n",
    "dispose(webb_sound)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ipytone.destination.volume.value = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}