Skip to content

Commit a19b599

Browse files
pedsonpedson
committed
Add multi-ccf export support.
1 parent cb6e527 commit a19b599

File tree

12 files changed

+280
-229
lines changed

12 files changed

+280
-229
lines changed

‎package.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,5 @@
3838
"gulp": "^4.0.2",
3939
"gulp-shell": "^0.8.0",
4040
"merge-stream": "^2.0.0"
41-
4241
}
4342
}

‎src/app.ts‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,20 @@
1+
import * as os from "os";
12
import {createServer} from "http";
23
import * as express from "express";
34
import * as bodyParser from "body-parser";
45

56
const debug = require("debug")("mnb:export-api:server");
67

78
import {ServiceOptions} from "./options/serviceOptions";
8-
9-
import {swcExportMiddleware} from "./middleware/swcExportMiddleware";
10-
import {jsonExportMiddleware} from "./middleware/jsonExportMiddleware";
11-
import * as os from "os";
9+
import {exportMiddleware} from "./middleware/exportMiddleware";
1210

1311
const app = express();
1412

1513
app.use(bodyParser.urlencoded({extended: true}));
1614

1715
app.use(bodyParser.json());
1816

19-
app.use("/swc", swcExportMiddleware);
20-
21-
app.use("/json", jsonExportMiddleware);
17+
app.use("/export", exportMiddleware);
2218

2319
const server = createServer(app);
2420

‎src/export/exportCacheBase.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import {MetricsStorageManager} from "../metricsStorageManager";
2+
3+
export enum ExportFormat {
4+
Swc = 0,
5+
Json = 1
6+
}
7+
8+
export enum CcfVersion {
9+
Janelia25,
10+
Aibs2017
11+
}
12+
13+
export interface IExportResponse {
14+
contents: any;
15+
filename: string;
16+
}
17+
18+
export abstract class ExportCacheBase {
19+
protected _cache = new Map<string, string>();
20+
21+
private readonly _ccfVersion: CcfVersion;
22+
23+
public get CcfVersion(): CcfVersion {
24+
return this._ccfVersion;
25+
}
26+
27+
private readonly _exportFormat: ExportFormat;
28+
29+
public get ExportFormat(): ExportFormat {
30+
return this._exportFormat;
31+
}
32+
33+
protected constructor(ccfVersion: CcfVersion, exportFormat: ExportFormat) {
34+
this._ccfVersion = ccfVersion;
35+
this._exportFormat = exportFormat;
36+
}
37+
38+
public abstract loadContents(): ExportCacheBase;
39+
40+
public findContents(ids: string[], hostname: string): Promise<IExportResponse> {
41+
return null;
42+
}
43+
44+
protected async logMetrics(ids: string[], hostname: string, duration: [number, number]): Promise<void> {
45+
await MetricsStorageManager.Instance().logExport({
46+
host: hostname,
47+
userId: "(unknown)",
48+
format: this.ExportFormat,
49+
ccfVersion: this.CcfVersion,
50+
ids: ids.join(", "),
51+
userName: "(unknown)",
52+
duration: duration[0] + duration[1] / 1000000000
53+
});
54+
}
55+
}

‎src/export/jsonExportCache.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import * as fs from "fs";
2+
import * as path from "path";
3+
import moment = require("moment");
4+
5+
import {ServiceOptions} from "../options/serviceOptions";
6+
import {CcfVersion, ExportFormat} from "./exportCacheBase";
7+
import {ExportCacheBase, IExportResponse} from "./exportCacheBase";
8+
9+
const debug = require("debug")("mnb:export-api:json");
10+
11+
export class JsonExportCache extends ExportCacheBase {
12+
public constructor(ccfVersion: CcfVersion) {
13+
super(ccfVersion, ExportFormat.Json);
14+
}
15+
16+
public loadContents(): ExportCacheBase {
17+
const dataLocation = path.join(ServiceOptions.dataPath, this.CcfVersion === CcfVersion.Janelia25 ? "json25" : "json30");
18+
19+
if (!fs.existsSync(dataLocation)) {
20+
debug("json data path does not exist");
21+
return;
22+
}
23+
24+
debug("initiating json cache load");
25+
26+
fs.readdirSync(dataLocation).forEach(file => {
27+
if (file.slice(-5) === ".json") {
28+
const jsonName = file.slice(0, -5);
29+
30+
const data = fs.readFileSync(path.join(dataLocation, file), {encoding: "utf8"});
31+
32+
this._cache.set(jsonName, JSON.parse(data).neuron);
33+
}
34+
});
35+
36+
debug(`loaded ${this._cache.size} neurons (json)`)
37+
38+
return this;
39+
}
40+
41+
public async findContents(ids: string[], hostname: string): Promise<IExportResponse> { const t1 = process.hrtime();
42+
if (!ids || ids.length === 0) {
43+
debug(`null json id request`);
44+
return;
45+
}
46+
47+
debug(`handling json request for ids: ${ids.join(", ")}`);
48+
49+
const base = {
50+
comment: `Downloaded ${moment().format("YYYY/MM/DD")}. Please consult Terms-of-Use at https://mouselight.janelia.org when referencing this reconstruction.`,
51+
neurons: []
52+
};
53+
54+
const contents = ids.reduce((prev, id) => {
55+
const data = this._cache.get(id);
56+
57+
if (data) {
58+
prev.neurons.push(data);
59+
}
60+
61+
return prev;
62+
}, base);
63+
64+
let filename = "mlnb-export.json";
65+
66+
if (ids.length === 1) {
67+
filename = ids[0] + ".json";
68+
}
69+
70+
await this.logMetrics(ids, hostname, process.hrtime(t1));
71+
72+
return {
73+
contents,
74+
filename
75+
};
76+
}
77+
}

‎src/export/swcExportCache.ts‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
import * as fs from "fs";
2+
import * as path from "path";
3+
import * as archiver from "archiver";
4+
import * as uuid from "uuid";
5+
import {ServiceOptions} from "../options/serviceOptions";
6+
import {CcfVersion, ExportFormat, ExportCacheBase, IExportResponse} from "./exportCacheBase";
7+
import moment = require("moment");
8+
9+
const debug = require("debug")("mnb:export-api:swc");
10+
11+
export class SwcExportCache extends ExportCacheBase {
12+
public constructor(ccfVersion: CcfVersion) {
13+
super(ccfVersion, ExportFormat.Swc);
14+
}
15+
16+
public loadContents(): ExportCacheBase {
17+
const dataLocation = path.join(ServiceOptions.dataPath, this.CcfVersion === CcfVersion.Janelia25 ? "swc25" : "swc30");
18+
19+
if (!fs.existsSync(dataLocation)) {
20+
debug("swc data path does not exist");
21+
return;
22+
}
23+
24+
debug("initiating swc cache load");
25+
26+
fs.readdirSync(dataLocation).forEach(file => {
27+
if (file.slice(-4) === ".swc") {
28+
const swcName = file.slice(0, -4);
29+
30+
const data = fs.readFileSync(path.join(dataLocation, file), {encoding: "utf8"});
31+
32+
this._cache.set(swcName, data);
33+
}
34+
});
35+
36+
debug(`loaded ${this._cache.size} neurons (swc)`)
37+
38+
return this;
39+
}
40+
41+
public async findContents(ids: string[], hostname: string): Promise<IExportResponse> {
42+
if (!ids || ids.length === 0) {
43+
debug(`null swc id request`);
44+
return null;
45+
}
46+
47+
const t1 = process.hrtime();
48+
49+
debug(`handling json request for ids: ${ids.join(", ")}`);
50+
51+
let response: IExportResponse;
52+
53+
if (ids.length === 1) {
54+
let encoded = null;
55+
56+
const data = this._cache.get(ids[0]);
57+
58+
if (data) {
59+
encoded = Buffer.from(`# Downloaded ${moment().format("YYYY/MM/DD")}. \n` + data).toString("base64");
60+
}
61+
62+
response = {
63+
contents: encoded,
64+
filename: ids[0] + ".swc"
65+
};
66+
} else {
67+
const tempFile = uuid.v4();
68+
69+
response = await new Promise(async (resolve) => {
70+
const output = fs.createWriteStream(tempFile);
71+
72+
output.on("finish", () => {
73+
const readData = fs.readFileSync(tempFile);
74+
75+
const encoded = readData.toString("base64");
76+
77+
fs.unlinkSync(tempFile);
78+
79+
resolve({
80+
contents: encoded,
81+
filename: "mlnb-export-data.zip"
82+
});
83+
});
84+
85+
const archive = archiver("zip", {zlib: {level: 9}});
86+
87+
archive.pipe(output);
88+
89+
ids.forEach(id => {
90+
const data = this._cache.get(id);
91+
92+
if (data) {
93+
archive.append(`# Generated ${moment().format("YYYY/MM/DD")}. \n` + data, {name: id + ".swc"});
94+
}
95+
});
96+
97+
await archive.finalize();
98+
});
99+
}
100+
101+
await this.logMetrics(ids, hostname, process.hrtime(t1));
102+
103+
return response;
104+
}
105+
}

‎src/metricsStorageManager.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,12 @@ import {FieldType, InfluxDB} from "influx";
33
const debug = require("debug")("mnb:export-api:metrics");
44

55
import {MetricsOptions} from "./options/databaseOptions";
6+
import {CcfVersion, ExportFormat} from "./export/exportCacheBase";
67

78
export interface IExportRequest {
8-
format: number,
9+
// @ts-ignore
10+
format: ExportFormat,
11+
ccfVersion: CcfVersion
912
ids: string;
1013
host: string;
1114
userName: string;
@@ -34,6 +37,7 @@ export class MetricsStorageManager {
3437
},
3538
fields: {
3639
format: exportRequest.format,
40+
ccfVersion: exportRequest.ccfVersion,
3741
ids: exportRequest.ids,
3842
userName: exportRequest.userName,
3943
duration: exportRequest.duration
@@ -76,6 +80,7 @@ async function establishConnection(): Promise<InfluxDB> {
7680
measurement: MetricsOptions.measurement,
7781
fields: {
7882
format: FieldType.INTEGER,
83+
ccfVersion: FieldType.INTEGER,
7984
ids: FieldType.STRING,
8085
userName: FieldType.STRING,
8186
duration: FieldType.FLOAT

‎src/middleware/exportFormat.ts‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎src/middleware/exportMiddleware.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import {CcfVersion, ExportFormat, ExportCacheBase} from "../export/exportCacheBase";
2+
import {JsonExportCache} from "../export/jsonExportCache";
3+
import {SwcExportCache} from "../export/swcExportCache";
4+
5+
const swcExport25 = (new SwcExportCache(CcfVersion.Janelia25)).loadContents();
6+
const swcExport30 = (new SwcExportCache(CcfVersion.Aibs2017)).loadContents();
7+
8+
const swcMap = new Map<CcfVersion, ExportCacheBase>();
9+
swcMap.set(CcfVersion.Janelia25, swcExport25);
10+
swcMap.set(CcfVersion.Aibs2017, swcExport30);
11+
12+
const jsonExport25 = (new JsonExportCache(CcfVersion.Janelia25)).loadContents();
13+
const jsonExport30 = (new JsonExportCache(CcfVersion.Aibs2017)).loadContents();
14+
15+
const jsonMap = new Map<CcfVersion, ExportCacheBase>();
16+
jsonMap.set(CcfVersion.Janelia25, jsonExport25);
17+
jsonMap.set(CcfVersion.Aibs2017, jsonExport30);
18+
19+
const map = new Map<ExportFormat, Map<CcfVersion, ExportCacheBase>>();
20+
map.set(ExportFormat.Swc, swcMap);
21+
map.set(ExportFormat.Json, jsonMap);
22+
23+
export async function exportMiddleware(req, res) {
24+
const format = req.body.format as ExportFormat ?? ExportFormat.Swc;
25+
26+
const ccfVersion = (req.body.ccfVersion as CcfVersion) ?? CcfVersion.Janelia25;
27+
28+
const source = (map.get(format)).get(ccfVersion);
29+
30+
const response = await source.findContents(req.body.ids, req.headers["x-forwarded-for"] || req.connection.remoteAddress)
31+
32+
res.json(response);
33+
}

0 commit comments

Comments
 (0)