fix: more rigorous handling of manifests, support for choosing column…

…s in download_to_depth

dev/FourClasses.svelte

@@ -0,0 +1,60 @@
+<script>
+  import { Scatterplot } from '../src/deepscatter';
+  import { onMount } from 'svelte';
+  import SwitchPositions from './svelte/SwitchPositions.svelte';
+  import ColorChange from './svelte/ColorChange.svelte';
+  import SizeSlider from './svelte/SizeSlider.svelte';
+
+  const startSize = 2;
+  const prefs = {
+    source_url: '/tiles',
+    max_points: 1000000,
+    alpha: 35, // Target saturation for the full page.
+    zoom_balance: 0.22, // Rate at which points increase size. https://observablehq.com/@bmschmidt/zoom-strategies-for-huge-scatterplots-with-three-js
+    point_size: startSize, // Default point size before application of size scaling
+    background_color: '#EEEDDE',
+    encoding: {
+      color: {
+        field: 'class',
+        range: 'category10',
+      },
+      x: {
+        field: 'x',
+        transform: 'literal',
+      },
+      y: {
+        field: 'y',
+        transform: 'literal',
+      },
+    },
+  };
+
+  let scatterplot = null;
+  onMount(() => {
+    scatterplot = new Scatterplot('#deepscatter');
+    window.scatterplot = scatterplot;
+    scatterplot.plotAPI(prefs);
+  });
+</script>
+
+<div id="overlay">
+  <SwitchPositions {scatterplot}></SwitchPositions>
+  <ColorChange {scatterplot}></ColorChange>
+  <SizeSlider size={startSize} {scatterplot}></SizeSlider>
+</div>
+
+<div id="deepscatter"></div>
+
+<style>
+  #overlay {
+    position: fixed;
+    z-index: 10;
+    left: 40px;
+    top: 40px;
+  }
+  #deepscatter {
+    z-index: 0;
+    width: 100vw;
+    height: 100vh;
+  }
+</style>

dev/Main.svelte

@@ -1,16 +1,28 @@
 <script>
+  import { onMount } from 'svelte';
   import FourClasses from './FourClasses.svelte';
   import SinglePoint from './SinglePoint.svelte';
   const modes = {
     FourClasses: FourClasses,
     SinglePoint: SinglePoint,
   };
 
-  $: mode = 'SinglePoint';
+  $: mode = '';
+  onMount(() => {
+    mode = window.location.hash.slice(1);
+    // window.onhashchange(() => {
+    //   mode = window.location.hash.slice(1);
+    // });
+  });
 </script>
 
 {#if mode in modes}
   <svelte:component this={modes[mode]} />
 {:else}
-  <h1>Mode not found</h1>
+  <h1>Put a load mode from the list in the hash.</h1>
+  <div>
+    {#each Object.keys(modes) as modename}
+      <a href="#{modename}">{modename}</a><br />
+    {/each}
+  </div>
 {/if}

dev/SinglePoint.svelte

@@ -0,0 +1,68 @@
+<script>
+  import { Table, tableFromArrays, tableFromJSON } from 'apache-arrow';
+  import { Scatterplot } from '../src/deepscatter';
+  import { onMount } from 'svelte';
+
+  const tb = tableFromArrays({
+    x: new Float32Array([0.001, 0.5, -0.5]),
+    y: new Float32Array([0.001, -0.5, 0.5]),
+    ix: new Int32Array([1, 2, 3]),
+  });
+
+  tb.schema.metadata.set(
+    'extent',
+    JSON.stringify({
+      x: [-1.1, 1.1],
+      y: [-1.1, 1.1],
+    }),
+  );
+
+  const startSize = 480;
+
+  const prefs = {
+    arrow_table: tb,
+    max_points: 1,
+    alpha: 100, // Target saturation for the full page.
+    zoom_balance: 0.22, // Rate at which points increase size. https://observablehq.com/@bmschmidt/zoom-strategies-for-huge-scatterplots-with-three-js
+    point_size: startSize, // Default point size before application of size scaling
+    background_color: '#EEEEEE',
+    encoding: {
+      color: {
+        constant: '#00FF00',
+      },
+      x: {
+        field: 'x',
+        transform: 'literal',
+      },
+      y: {
+        field: 'y',
+        transform: 'literal',
+      },
+    },
+  };
+
+  let scatterplot = null;
+  onMount(() => {
+    scatterplot = new Scatterplot('#deepscatter', 480, 480);
+    window.scatterplot = scatterplot;
+    scatterplot.plotAPI(prefs);
+  });
+</script>
+
+<div id="overlay"></div>
+
+<div id="deepscatter"></div>
+
+<style>
+  #overlay {
+    position: fixed;
+    z-index: -10;
+    left: 40px;
+    top: 40px;
+  }
+  #deepscatter {
+    z-index: 0;
+    width: 100vw;
+    height: 100vh;
+  }
+</style>

package.json

@@ -19,9 +19,11 @@
   "scripts": {
     "dev": "vite --mode dev --port 3344 --host",
     "format": "prettier --write src",
+
     "build": "vite build && tsc",
     "prepublishOnly": "vite build && tsc && typedoc --skipErrorChecking src/*",
-    "test": "playwright test",
+    "test": "vite build && npm run test:node",
+    "test:playwright": "playwright test",
     "test:node": "node tests/dataset.spec.js",
     "lint": "eslint src",
     "docs": "typedoc --out docs src/deepscatter.ts"

src/Dataset.ts

@@ -62,7 +62,7 @@ const defaultTransformations: Record<string, Transformation> = {
 export class Dataset {
   public transformations: Record<string, Transformation> =
     defaultTransformations;
-  protected plot: Scatterplot;
+  public _plot: Scatterplot | null;
   private extents: Record<string, [number, number] | [Date, Date]> = {};
   // A 3d identifier for the tile. Usually [z, x, y]
   private _extent?: Rectangle;
@@ -83,10 +83,10 @@ export class Dataset {
 
   constructor(
     base_url: string,
-    plot: Scatterplot,
+    plot: Scatterplot | null = null,
     options: DS.DatasetOptions = {},
   ) {
-    this.plot = plot;
+    this._plot = plot;
     this.tileProxy = options.tileProxy;
 
     const rootKey = options['rootKey'] || '0/0/0';
@@ -115,10 +115,16 @@ export class Dataset {
     this.promise = preProcessRootTile.then(async () => {
       const batch = await this.root_tile.get_arrow(null);
       const schema = batch.schema;
-      await this.root_tile.loadManifestInfoFromTileMetadata();
+      console.log('BBB HERE');
+      this.root_tile.manifest =
+        await this.root_tile.deriveManifestInfoFromTileMetadata();
+      console.log('BBBB HERE');
+
       if (schema.metadata.has('sidecars')) {
         const cars = schema.metadata.get('sidecars');
+        console.log('BBB HERE');
         const parsed = JSON.parse(cars as string) as Record<string, string>;
+        console.log('HERE');
         for (const [k, v] of Object.entries(parsed)) {
           this.transformations[k] = async function (tile) {
             const batch = await tile.get_arrow(v);
@@ -136,9 +142,18 @@ export class Dataset {
       } else {
         // "NO SIDECARS"
       }
+      console.log(this.root_tile.manifest);
     });
   }
 
+  get plot() {
+    if (this._plot === undefined) {
+      throw new Error('Plot not yet bound');
+    } else {
+      return this._plot;
+    }
+  }
+
   get ready() {
     return this.promise;
   }
@@ -156,7 +171,7 @@ export class Dataset {
     if (!this.root_tile.hasLoadedColumn('x')) {
       throw new Error("Can't access extent without a root tile");
     }
-    return this.root_tile.extent;
+    return this.root_tile.manifest.extent;
   }
 
   /**
@@ -256,7 +271,7 @@ export class Dataset {
    * @param plot The Scatterplot to use.
    * @returns
    */
-  static from_arrow_table(table: Table, plot: Scatterplot): Dataset {
+  static fromArrowTable(table: Table, plot: Scatterplot): Dataset {
     return wrapArrowTable(tableToIPC(table), plot);
   }
 
@@ -451,13 +466,13 @@ export class Dataset {
 
     let seen = 0;
     const start = starting_tile || this.root_tile;
-    await start.loadManifestInfoFromTileMetadata();
+    await start.deriveManifestInfoFromTileMetadata();
     const total_points = JSON.parse(
       start.record_batch.schema.metadata.get('total_points'),
     ) as number;
 
     async function resolve(tile: Tile) {
-      await tile.loadManifestInfoFromTileMetadata();
+      await tile.deriveManifestInfoFromTileMetadata();
       if (!filter(tile)) {
         return;
       }
@@ -616,6 +631,7 @@ export class Dataset {
     bbox: Rectangle | undefined,
     max_ix: number,
     queue_length = 8,
+    fields: string[] = ['x', 'y', 'ix'],
   ) {
     /*
       Browsing can spawn a  *lot* of download requests that persist on
@@ -655,10 +671,16 @@ export class Dataset {
       if ((tile.min_ix && tile.min_ix > max_ix) || distance <= 0) {
         continue;
       }
+      console.log(tile.key);
       queue.add(tile.key);
-      tile
-        .loadManifestInfoFromTileMetadata()
-        .then(() => queue.delete(tile.key))
+
+      Promise.all([
+        tile.populateManifest(),
+        ...fields.map((fieldname) => tile.get_column(fieldname)),
+      ])
+        .then(() => {
+          queue.delete(tile.key);
+        })
         .catch((error) => {
           console.warn('Error on', tile.key);
           queue.delete(tile.key);
@@ -780,8 +802,7 @@ export function add_or_delete_column(
         // Then it's dropped.
         continue;
       } else {
-        console.warn(`Name ${field.name} already exists, can't add.`);
-        return batch as RecordBatch;
+        return batch;
       }
     }
     const current = batch.getChild(field.name);