Node testing setup (#34)

Author: kylebarron

.github/workflows/test.yml

@@ -27,6 +27,27 @@ jobs:
       # - run: cargo check                                 --no-default-features --features "console_error_panic_hook wee_alloc"
       # - run: cargo check --target wasm32-unknown-unknown --no-default-features --features "console_error_panic_hook wee_alloc"
 
+  node-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: "16"
+
+      - name: Build bundle
+        run: bash ./scripts/build.sh
+
+      - name: Install dev dependencies
+        run: yarn
+
+      - name: Run Node tests
+        run: yarn test
+
   fmt:
     name: fmt
     runs-on: ubuntu-latest

data/data.arrow

@@ -0,0 +1,13 @@
+{
+  "scripts": {
+    "test": "ts-node node_modules/tape/bin/tape ./tests/js/index.ts"
+  },
+  "devDependencies": {
+    "@types/node": "^17.0.21",
+    "@types/tape": "^4.13.2",
+    "apache-arrow": "^7.0.0",
+    "tape": "^5.5.2",
+    "ts-node": "^10.7.0",
+    "typescript": "^4.6.2"
+  }
+}

data/not_work.parquet

@@ -2,7 +2,7 @@
 import pyarrow.feather as feather
 import pyarrow.parquet as pq
 
-compressions = ["SNAPPY", "GZIP", "BROTLI", "LZ4", "ZSTD", 'NONE']
+compressions = ["SNAPPY", "GZIP", "BROTLI", "LZ4", "ZSTD", "NONE"]
 
 
 def create_data():
@@ -35,19 +35,5 @@ def main():
     write_data(table)
 
 
-def debug():
-    """Create debug data"""
-    works = pa.table({
-        "uint8": pa.array([1, 2, 3, 4], type=pa.uint8()),
-    })
-    pq.write_table(works, 'works.parquet', compression='NONE')
-
-    not_work = pa.table({
-        "int8": pa.array([0, 0, 0, 0], type=pa.int8()),
-    })
-    pq.write_table(not_work, 'not_work.parquet', compression='NONE')
-
-
 if __name__ == "__main__":
     main()
-    debug()

data/works.parquet

@@ -0,0 +1,60 @@
+import * as test from "tape";
+import * as wasm from "../../pkg/node";
+import { readFileSync } from "fs";
+import { tableFromIPC, tableToIPC, Table } from "apache-arrow";
+
+// Path from repo root
+const dataDir = "tests/data";
+const testFiles = [
+  "1-partition-brotli.parquet",
+  "1-partition-gzip.parquet",
+  // "1-partition-lz4.parquet",
+  "1-partition-none.parquet",
+  "1-partition-snappy.parquet",
+  // "1-partition-zstd.parquet",
+  "2-partition-brotli.parquet",
+  "2-partition-gzip.parquet",
+  // "2-partition-lz4.parquet",
+  "2-partition-none.parquet",
+  "2-partition-snappy.parquet",
+  // "2-partition-zstd.parquet",
+];
+
+function testArrow(t: test.Test, table: Table) {
+  t.equals(table.numRows, 4, "correct number of rows");
+  t.deepEquals(
+    table.schema.fields.map((f) => f.name),
+    ["str", "uint8", "int32", "bool"],
+    "correct column names"
+  );
+}
+
+test("read file", async (t) => {
+  for (const testFile of testFiles) {
+    const dataPath = `${dataDir}/${testFile}`;
+    const buffer = readFileSync(dataPath);
+    const arr = new Uint8Array(buffer);
+    const table = tableFromIPC(wasm.readParquet(arr));
+    testArrow(t, table);
+  }
+
+  t.end();
+});
+
+test("write and read file", async (t) => {
+  const dataPath = `${dataDir}/1-partition-brotli.parquet`;
+  const buffer = readFileSync(dataPath);
+  const arr = new Uint8Array(buffer);
+  const initialTable = tableFromIPC(wasm.readParquet(arr));
+
+  const writerProperties = new wasm.WriterPropertiesBuilder().build();
+
+  const parquetBuffer = wasm.writeParquet(
+    tableToIPC(initialTable, "stream"),
+    writerProperties
+  );
+  const table = tableFromIPC(wasm.readParquet(parquetBuffer));
+
+  testArrow(t, table);
+  t.end();
+});

package.json

@@ -0,0 +1 @@
+import './arrow1';

data/1-partition-brotli.parquet tests/data/1-partition-brotli.parquet

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "moduleResolution": "node"
+  },
+  "include": [
+    "tests/**/*"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}