03-Hubble.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d4809ac8-a92d-451d-9690-8f7b56022019",
   "metadata": {},
   "source": [
    "# Hubble Deep Space Sonification\n",
    "\n",
    "Example inspired from https://www.nasa.gov/content/explore-from-space-to-sound"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6377cae8-4142-4d98-b9be-dcd8691adfe9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ipytone\n",
    "import skimage\n",
    "import numpy as np\n",
    "import ipywidgets as widgets\n",
    "import bqplot\n",
    "import bqplot.pyplot as bqp\n",
    "from IPython.display import YouTubeVideo, Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "525d694d-917d-4867-8228-5e95ed57178d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "YouTubeVideo(\"H-Ci_YwfH04\", width=600, height=400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2031de5d-7b81-4b55-960e-b2d4869096f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "image = skimage.data.hubble_deep_field()\n",
    "\n",
    "skimage.io.imshow(image);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d5738f3-143a-4e13-b11d-0d9937f75802",
   "metadata": {},
   "source": [
    "1. Create a fixed number of oscillators at evenly spaced frequencies\n",
    "2. Connect the oscillators to gain nodes (controlled by image data)\n",
    "3. Connect all gain nodes to a `Panner` node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "875fb790-1a61-4f22-8bf4-d7fc9b6d08d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_oscillators = 40\n",
    "freq_min = 50\n",
    "freq_max = 1000\n",
    "\n",
    "freqs = np.linspace(freq_min, freq_max, n_oscillators)\n",
    "\n",
    "oscillators = []\n",
    "gains = []\n",
    "\n",
    "pan = ipytone.Panner().to_destination()\n",
    "\n",
    "for freq in freqs:\n",
    "    osc = ipytone.Oscillator(frequency=freq, volume=-15)\n",
    "    gain = ipytone.Gain(gain=0)\n",
    "    osc.chain(gain, pan)\n",
    "    \n",
    "    oscillators.append(osc)\n",
    "    gains.append(gain)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "500b1b19-4245-4a97-82a3-ff42c98ee438",
   "metadata": {},
   "source": [
    "4. Resize the image (number of rows = number of oscillators)\n",
    "5. Extract intensity (used to control the gain nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25358857-c436-48e6-a9b0-5980f8548c58",
   "metadata": {},
   "outputs": [],
   "source": [
    "# resize\n",
    "new_shape = [n_oscillators, image.shape[1]]\n",
    "resized = skimage.transform.resize(image, new_shape)\n",
    "\n",
    "# get intensity\n",
    "intensity = skimage.color.rgb2gray(resized)\n",
    "\n",
    "# remove noise\n",
    "intensity = np.where(intensity > 0.1, intensity, 0)\n",
    "\n",
    "# will start and stop with all gains set to zero\n",
    "intensity = np.pad(intensity, pad_width=((0, 0), (1, 1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e809548c-933c-48ab-b493-84c974aec796",
   "metadata": {},
   "source": [
    "6. Generate the automation curves (gains, panning, etc.) for a given time range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb95a0d2-4af8-46f5-b4e6-10cb34407a79",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = ipytone.get_transport()\n",
    "\n",
    "duration = 30   # total animation duration (seconds)\n",
    "\n",
    "def osc_start_stop(time):\n",
    "    for osc in oscillators:\n",
    "        osc.start(time).stop(time + duration)\n",
    "\n",
    "def gain_automation(time):\n",
    "    for i in range(1, n_oscillators + 1):\n",
    "        gains[i - 1].gain.set_value_curve_at_time(intensity[-i], time, duration)\n",
    "\n",
    "def pan_automation(time):\n",
    "    pan.pan.set_value_at_time(-1, time)\n",
    "    pan.pan.linear_ramp_to_value_at_time(1, time + duration)\n",
    "\n",
    "osc_eid = t.schedule(osc_start_stop, 0)\n",
    "gain_eid = t.schedule(gain_automation, 0)\n",
    "pan_eid = t.schedule(pan_automation, 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1159ec5e-8dbd-4477-bb27-8d13bffc1829",
   "metadata": {},
   "source": [
    "7. Create the interactive figure with `bqplot`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4d59533-c0c1-42ec-a683-b2572406cb30",
   "metadata": {},
   "outputs": [],
   "source": [
    "skimage.io.imsave(\"temp.png\", image)\n",
    "\n",
    "with open(\"temp.png\", \"rb\") as f:\n",
    "    raw_image = f.read()\n",
    "\n",
    "ipyimage = widgets.Image(value=raw_image, format=\"png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2dd2a48-b26f-4934-a418-17e04370bea1",
   "metadata": {},
   "outputs": [],
   "source": [
    "scales = {\n",
    "    \"x\": bqplot.LinearScale(min=0, max=duration),\n",
    "    \"y\": bqplot.LinearScale(min=freq_min, max=freq_max)\n",
    "}\n",
    "\n",
    "bqimage = bqplot.Image(\n",
    "    image=ipyimage,\n",
    "    scales=scales,\n",
    "    x=(0, 30),\n",
    "    y=(freq_min, freq_max)\n",
    ")\n",
    "\n",
    "lines = bqplot.Lines(\n",
    "    x=np.zeros(n_oscillators) + 0.25,\n",
    "    y=freqs,\n",
    "    scales=scales,\n",
    ")\n",
    "lines.colors = [\"white\"]\n",
    "\n",
    "vline = bqp.vline(0, scales=scales)\n",
    "vline.colors = [\"white\"]\n",
    "\n",
    "fig = bqplot.Figure(marks=[bqimage, lines, vline])\n",
    "fig.layout.width = \"500px\"\n",
    "fig.layout.height = \"500px\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39bd94e5-e8fb-458b-8109-17ef02e08a16",
   "metadata": {},
   "source": [
    "8. Moving vertical line (cursor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64dfe0c9-b161-41cc-b6c8-3c40b96971aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "def move_lines(change):\n",
    "    time = change[\"new\"]\n",
    "    vline.x = [time, time]\n",
    "    \n",
    "    idx = int(intensity.shape[1] / duration * time)\n",
    "    lines.x = time + 0.25 + intensity[::-1, idx] * 5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6047a428-f1ec-4507-87bc-8391debfdb66",
   "metadata": {},
   "outputs": [],
   "source": [
    "t.schedule_observe(\n",
    "    move_lines, update_interval=0.1, name=\"seconds\", transport=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df49b976-a8e3-4da7-ac39-5c1d51c04dfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff7382bb-9d3c-4810-9697-fdfabb5b7be8",
   "metadata": {},
   "source": [
    "9. Let's see and hear the result!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e53f10f9-5918-4f03-a8ad-de75f0de008b",
   "metadata": {},
   "outputs": [],
   "source": [
    "t.start().stop(f\"+{duration}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a1f9fa7-644c-4ad4-9448-6fa0d57eefad",
   "metadata": {},
   "source": [
    "- Clean-up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c237aeff-50f3-43ae-a5a4-f8692c202a25",
   "metadata": {},
   "outputs": [],
   "source": [
    "t.schedule_unobserve(move_lines)\n",
    "t.clear(osc_eid).clear(gain_eid).clear(pan_eid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb3482d7-b546-49d0-90d0-5125f994906e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for osc in oscillators:\n",
    "    osc.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34f6f219-7f15-460f-86be-58bf2861298e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for osc in oscillators:\n",
    "    osc.close()\n",
    "\n",
    "for gain in gains:\n",
    "    gain.close()\n",
    "    \n",
    "pan.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd4baa60-b436-4407-a281-f66723ccb03e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}