From 39cee7e8dd206fff13e95215ca737403cbb31895 Mon Sep 17 00:00:00 2001 From: Mike Kucera Date: Thu, 7 Nov 2024 17:02:40 -0500 Subject: [PATCH] Save motifs and tracks in mongo --- .../components/network-editor/controller.js | 2 +- src/client/components/network-editor/index.js | 4 +- src/client/components/report/report.js | 13 +- src/server/datastore.js | 788 ++++-------------- src/server/env.js | 1 - src/server/index.js | 2 +- src/server/routes/api/create.js | 178 ---- src/server/routes/api/export.js | 53 +- src/server/routes/api/index.js | 165 ++-- src/server/routes/api/report.js | 4 +- 10 files changed, 228 insertions(+), 982 deletions(-) diff --git a/src/client/components/network-editor/controller.js b/src/client/components/network-editor/controller.js index e6d6c304..fa807faa 100644 --- a/src/client/components/network-editor/controller.js +++ b/src/client/components/network-editor/controller.js @@ -503,7 +503,7 @@ export class NetworkEditorController { fetch(`/api/${this.networkIDStr}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ networkName }) + body: JSON.stringify({ name: networkName }) }); } diff --git a/src/client/components/network-editor/index.js b/src/client/components/network-editor/index.js index c4679059..b269bf1d 100644 --- a/src/client/components/network-editor/index.js +++ b/src/client/components/network-editor/index.js @@ -69,11 +69,9 @@ async function loadNetwork(id, cy, controller, recentNetworksController) { } const networkJson = await networkResult.json(); - cy.add(networkJson.network.elements); cy.data({ - name: networkJson.networkName, + name: networkJson.name, parameters: networkJson.parameters, - geneSetCollection: networkJson.geneSetCollection, demo: Boolean(networkJson.demo) }); diff --git a/src/client/components/report/report.js b/src/client/components/report/report.js index 69e8518e..8f95b92c 100644 --- a/src/client/components/report/report.js +++ b/src/client/components/report/report.js @@ -1,6 +1,6 @@ import React, { useState, useEffect } from 'react'; import PropTypes from 'prop-types'; -import { getComparator } from '../network-editor/data-table'; +import { getComparator } from '../util'; import { Select, MenuItem } from '@mui/material'; import { Table, TableBody, TableCell, TableContainer, TableHead, TableRow, Paper } from '@mui/material'; import makeStyles from '@mui/styles/makeStyles'; @@ -31,6 +31,7 @@ async function fetchReport(secret) { const counts = await countRes.json(); const networks = await networkRes.json(); + console.log(counts, networks); return { counts, networks }; } catch(err) { @@ -68,7 +69,7 @@ export function Report({ secret }) { Sort:    @@ -84,9 +85,6 @@ export function Report({ secret }) { Network Name - Nodes - Edges - Type Creation Time Last Access Time @@ -101,10 +99,7 @@ export function Report({ secret }) { key={network._id} sx={{ '&:last-child td, &:last-child th': { border: 0 } }} > - {network.networkName} - {network.nodeCount} - {network.edgeCount} - {network.inputType} + {network.name} {createTime} {accessTime} open diff --git a/src/server/datastore.js b/src/server/datastore.js index 6f0ad880..03cdf6f3 100644 --- a/src/server/datastore.js +++ b/src/server/datastore.js @@ -3,15 +3,22 @@ import uuid from 'uuid'; import MUUID from 'uuid-mongodb'; import _ from 'lodash'; import fs from 'fs'; -import { fileForEachLine, parseMotifsAndTracks, annotateGenes } from './util.js'; +import { parseMotifsAndTracks, annotateGenes } from './util.js'; -const GENE_RANKS_COLLECTION = 'geneRanks'; -const GENE_LISTS_COLLECTION = 'geneLists'; -const NETWORKS_COLLECTION = 'networks'; -const PERFORMANCE_COLLECTION = 'performance'; -const POSITIONS_COLLECTION = 'positions'; -const POSITIONS_RESTORE_COLLECTION = "positionsRestore"; +// This is the promary collection, each document represents the results +// of an analysis returned by the iregulon service. Documens in this collection +// are created once and are (mostly) static. +const MOTIFS_AND_TRACKS_COLLECTION = 'motifsAndTracks'; + +// This collection contains state data that is associated with a document in the +// motifsAndTracks collection. It contains mutable state data like the name +// of the document and UI state (like whats selected in the data table). +const STATE_DATA_COLLECTION = 'stateData'; + +// const PERFORMANCE_COLLECTION = 'performance'; + +export const DEMO_ID = '7cea4157-341a-4fc6-b6c4-9c7ac5bcc8d4'; /** @@ -29,72 +36,31 @@ function makeID(strOrObj) { } -/** - * Converts a ranked gene list in TSV format into the document - * format we want for mongo. - */ -export function rankedGeneListToDocument(rankedGeneList, delimiter = '\t') { - const genes = []; - var [min, max] = [Infinity, -Infinity]; - - rankedGeneList.split("\n").slice(1).forEach(line => { - const [gene, rankStr] = line.split(delimiter); - - const rank = Number(rankStr); - - if (gene) { - if (isNaN(rank)) { - genes.push({ gene }); - } else { - min = Math.min(min, rank); - max = Math.max(max, rank); - genes.push({ gene, rank }); - } - } - }); - - return { genes, min, max }; -} - - -/** - * Converts a JSON object in the format { "GENENAME1": rank1, "GENENAME2": rank2, ... } - * into the document format we want for mongo. - */ -export function fgseaServiceGeneRanksToDocument(rankedGeneListObject) { - const genes = []; - var [min, max] = [Infinity, -Infinity]; - - for(const [gene, rank] of Object.entries(rankedGeneListObject)) { - min = Math.min(min, rank); - max = Math.max(max, rank); - genes.push({ gene, rank }); - } - - return { genes, min, max }; -} - - - class Datastore { // mongo; // mongo connection obj // db; // app db - // queries; // queries collection (i.e. query results) - - // TODO Remove this--just for prototyping =================== - saveResults(genes, results) { - const networkID = makeID(); - this.RESULTS_CACHE.set(networkID.string, { genes, network: { elements: [] }, results }); - - return networkID.string; + constructor() { } + + async initialize() { + await this.connect(); + await this.createIndexes(); + await this.loadDemo(); } - //=========================================================== + async connect() { + console.info('Connecting to MongoDB'); + const { MONGO_URL, MONGO_ROOT_NAME } = process.env; + this.mongo = await MongoClient.connect(MONGO_URL, { useNewUrlParser: true, useUnifiedTopology: true }); + this.db = this.mongo.db(MONGO_ROOT_NAME); + console.info('Connected to MongoDB'); + } - constructor() { - // TODO Remove this--just for prototyping =================== - this.RESULTS_CACHE = new Map(); + async loadDemo() { + if(await this.idExists(DEMO_ID)) { + console.log("- Demo Results Already Loaded"); + return; + } // Load demo file (public/sample-data/hypoxia_geneset-results.tsv) into cache fs.readFile('public/sample-data/hypoxia_geneset-results.tsv', 'utf8', (err, data) => { @@ -115,598 +81,184 @@ class Datastore { annotateGenes(genes, results); console.log("- Demo Genes Loaded:", genes.length); - // Store the results in the cache - this.RESULTS_CACHE.set( - '7cea4157-341a-4fc6-b6c4-9c7ac5bcc8d4', - { genes: genes, network: { elements: [] }, results } - ); + this.saveResults(genes, results, "Demo Network", DEMO_ID); }); }); - //=========================================================== - } - - async connect() { - console.info('Connecting to MongoDB'); - const { MONGO_URL, MONGO_ROOT_NAME, MONGO_COLLECTION_QUERIES } = process.env; - - const mongo = this.mongo = await MongoClient.connect(MONGO_URL, { useNewUrlParser: true, useUnifiedTopology: true }); - const db = this.db = mongo.db(MONGO_ROOT_NAME); - this.queries = db.collection(MONGO_COLLECTION_QUERIES); - console.info('Connected to MongoDB'); - } - - async initializeGeneSetDB(dbFilePath, dbFileName) { - console.info('Initializing MongoDB...'); - await this.loadGenesetDB(dbFilePath, dbFileName); - console.info('iRegulon Database Loaded: ' + dbFileName); - await this.createIndexes(); - console.info('Indexes Created'); - console.info('MongoDB Intialization Done'); - } - - async dropCollectionIfExists(name) { - const collections = await this.db.listCollections().toArray(); - console.log(`Checking if collection '${name}' exists`); - if(collections.some(c => c.name === name)) { - console.log(` It does! Dropping '${name}'`); - await this.db.collection(name).drop(); - } } /** - * @param dbFileName Name of the GMT file. Use one of the constants at the top of this file. + * @returns The id of the created document. */ - async loadGenesetDB(path, dbFileName) { - const isLoaded = async () => { - const collections = await this.db.listCollections().toArray(); - return collections.some(c => c.name === dbFileName); - }; - - if(await isLoaded()) { - console.info("Collection " + dbFileName + " already loaded"); - return; - } else { - console.info("Loading collection " + dbFileName); - } + async saveResults(genes, results, name, demoID) { + name = name || "Untitled Network"; + const id = demoID ? makeID(demoID) : makeID(); + + const motifsAndTracksDocument = { + _id: id.bson, + genes, + results, // motifs and tracks + creationTime: new Date(), + demo: Boolean(demoID), + }; + + const stateDocument = { + name, + motifsAndTracksID: id.bson, + }; - // Create indexes on dbFileName collection first await this.db - .collection(dbFileName) - .createIndex({ name: 1 }, { unique: true }); + .collection(MOTIFS_AND_TRACKS_COLLECTION) + .insertOne(motifsAndTracksDocument); await this.db - .collection(dbFileName) - .createIndex({ genes: 1 }); - - - const filepath = path + dbFileName; - const writeOps = []; - - await fileForEachLine(filepath, line => { - const [name, description, ...genes] = line.split("\t"); - if(genes[genes.length - 1] === "") { - genes.pop(); - } + .collection(STATE_DATA_COLLECTION) + .insertOne(stateDocument); - writeOps.push({ - updateOne: { - filter: { name }, // index on name already created - update: { $set: { name, description, genes } }, - upsert: true - } - }); - - }); - - await this.db - .collection(dbFileName) - .bulkWrite(writeOps); + return id.string; } - - + + async createIndexes() { await this.db - .collection(GENE_LISTS_COLLECTION) - .createIndex({ networkID: 1 }); - - // TODO is this index (into an array) still necessary?? - await this.db - .collection(GENE_LISTS_COLLECTION) - .createIndex({ 'genes.gene': 1 }); - - await this.db - .collection(GENE_RANKS_COLLECTION) - .createIndex({ networkID: 1 }); - - await this.db - .collection(POSITIONS_COLLECTION) - .createIndex({ networkID: 1 }); - - await this.db - .collection(POSITIONS_RESTORE_COLLECTION) - .createIndex({ networkID: 1 }); - - await this.db - .collection(GENE_RANKS_COLLECTION) - .createIndex({ networkID: 1, gene: 1 }, { unique: true }); - - await this.db - .collection(NETWORKS_COLLECTION) - .createIndex({ demo: 1 }); + .collection(STATE_DATA_COLLECTION) + .createIndex({ motifsAndTracksID: 1 }); } /** - * Inserts a network document into the 'networks' collection. - * @returns The id of the created document. + * Returns the motifs and tracks document. */ - async createNetwork(networkJson, networkName, type, geneSetCollection, demo) { - if (typeof (networkJson) == 'string') { - networkJson = JSON.parse(networkJson); + async getMotifsAndTracks(idStr) { + let id; + try { + id = makeID(idStr); + } catch { + console.log(`Invalid ID: '${idStr}'`); + return null; } - const networkID = makeID(); - - networkJson['_id'] = networkID.bson; - networkJson['networkIDStr'] = networkID.string; - networkJson['creationTime'] = new Date(); - - delete networkJson['summaryNetwork']; - - if(networkName) - networkJson['networkName'] = networkName; - if(type) - networkJson['inputType'] = type; - if(geneSetCollection) - networkJson['geneSetCollection'] = geneSetCollection; - if(demo) - networkJson['demo'] = true; - - await this.db - .collection(NETWORKS_COLLECTION) - .insertOne(networkJson); - - return networkID.string; - } - - /** - * Updates a network document--only the 'networkName' can be updated. - * @returns true if the network has been found and updated, false otherwise. - */ - async updateNetwork(networkIDString, { networkName }) { - const networkID = makeID(networkIDString); - - const res = await this.db - .collection(NETWORKS_COLLECTION) - .updateOne( - { '_id': networkID.bson }, - { $set: { networkName: networkName } } + const docResult = await this.db + .collection(MOTIFS_AND_TRACKS_COLLECTION) + .findOneAndUpdate( + { _id: id.bson }, + { $set: { lastAccessTime: new Date() } }, + { returnDocument: 'after' } ); - return res.modifiedCount > 0; - } - - /** - * Inserts the given document into the 'performance' collection. - */ - async createPerfDocument(networkIDString, document) { - if(networkIDString) { - document = { - networkID: makeID(networkIDString).bson, - ...document - }; + if(!docResult) { + return null; } - await this.db - .collection(PERFORMANCE_COLLECTION) - .insertOne(document); - } + const motifsAndTracks = docResult.value; + const stateResult = await this.db + .collection(STATE_DATA_COLLECTION) + .findOne({ motifsAndTracksID: id.bson }); - /** - * Creates individual documents in the gene ranks collection. - * This must be called before the mergeXXX functions. - */ - async createGeneRanksDocuments(networkID) { - // Create a collection with { networkID, gene } as the key, for quick lookup of gene ranks. - await this.db - .collection(GENE_LISTS_COLLECTION) - .aggregate([ - { $match: { networkID: networkID.bson } }, - { $project: { _id: 0, networkID: 1, genes: 1 } }, - { $unwind: "$genes" }, - { $project: { networkID: 1, gene: "$genes.gene", rank: "$genes.rank" } }, - { $merge: { - into: GENE_RANKS_COLLECTION, - on: [ "networkID", "gene" ] - } - } - ]).toArray(); - } - - - async getDemoNetworkIDs() { - const ids = await this.db - .collection(NETWORKS_COLLECTION) - .find({ demo: true }, { projection: { _id: 0, networkIDStr: 1 } } ) - .sort({ creationTime: 1 }) - .map(obj => obj.networkIDStr) - .toArray(); - - return ids; - } - - - /** - * Returns the network document. - */ - async getNetwork(networkIDString) { - console.log("-- Fetching network: ", networkIDString); - console.log(this.RESULTS_CACHE.keys()); - return this.RESULTS_CACHE.get(networkIDString); - - // let networkID; - // try { - // networkID = makeID(networkIDString); - // } catch { - // console.log(`Invalid network ID: '${networkIDString}'`); - // return null; - // } - - // const result = await this.db - // .collection(NETWORKS_COLLECTION) - // .findOneAndUpdate( - // { _id: networkID.bson }, - // { $set: { lastAccessTime: new Date() } }, - // { returnDocument: 'after', - // // projection: { network: false } - // } - // ); - - // if(!result) { - // return null; - // } - // const network = result.value; - // return network; - } - - - /** - * { - * _id: 'asdf', - * networkID: "abcdefg", - * positions: [ - * { - * id: "asdf-asdf-asdf", - * x: 12.34 - * y: 56.78 - * collapsed: false - * } - * ] - * } - * Note: Deleted nodes are not part of the positions document. - */ - async setPositions(networkIDString, positions) { - const networkID = makeID(networkIDString); - const document = { - networkID: networkID.bson, - positions - }; - - // Save the document twice, the one in POSITIONS_RESTORE_COLLECTION never changes - await this.db - .collection(POSITIONS_RESTORE_COLLECTION) - .updateOne({ networkID: networkID.bson }, { $setOnInsert: document }, { upsert: true }); - - await this.db - .collection(POSITIONS_COLLECTION) - .replaceOne({ networkID: networkID.bson }, document, { upsert: true }); - } - - async getPositions(networkIDString) { - const networkID = makeID(networkIDString); - - let result = await this.db - .collection(POSITIONS_COLLECTION) - .findOne({ networkID: networkID.bson }); - - if(!result) { - console.log("Did not find positions, querying POSITIONS_RESTORE_COLLECTION"); - result = await this.db - .collection(POSITIONS_RESTORE_COLLECTION) - .findOne({ networkID: networkID.bson }); + if(stateResult) { + motifsAndTracks.name = stateResult.name; } - - return result; + return motifsAndTracks; } - async deletePositions(networkIDString) { - const networkID = makeID(networkIDString); - // Only delete from POSITIONS_COLLECTION, not from POSITIONS_RESTORE_COLLECTION - await this.db - .collection(POSITIONS_COLLECTION) - .deleteOne({ networkID: networkID.bson } ); - } - - - - /** - * Returns the aggregation pipeline stages needed to extract - * the FGSEA enrichment results from the NETWORKS_COLLECTION. - * - * The results are of the form... - * - * { - * "padj": 0, - * "NES": -1.8082, - * "name": "MITOTIC METAPHASE AND ANAPHASE%REACTOME%R-HSA-2555396.2", - * "pval": 5.6229e-7, - * "size": 229 - * } - */ - _enrichmentQuery(networkID) { - return [ - { $match: { _id: networkID.bson } }, - { $replaceWith: { path: "$network.elements.nodes.data" } }, - { $unwind: { path: "$path" } }, - { $replaceRoot: { newRoot: "$path" } }, - { $project: { - name: { $arrayElemAt: [ "$name", 0 ] }, - pval: "$pvalue", - padj: true, - NES: true, - size: "$gs_size", - mcode_cluster_id: true - }} - ]; - } - - /** - * Returns an cursor of renrichment results objects. - */ - async getEnrichmentResultsCursor(networkIDString) { - const networkID = makeID(networkIDString); - - const cursor = await this.db - .collection(NETWORKS_COLLECTION) - .aggregate(this._enrichmentQuery(networkID)); - - return cursor; + async getGenesForSearch(idStr) { + const results = await this.getMotifsAndTracks(idStr); + return results.genes; } - /** - * Returns an cursor of objects of the form (sorted by rank): - * [ { "gene": "ABCD", "rank": 0.0322 }, ... ] - */ - async getRankedGeneListCursor(networkIDString) { - const networkID = makeID(networkIDString); + async getResultsForSearch(idStr) { + const results = await this.getMotifsAndTracks(idStr); + return results.results; + } - const cursor = await this.db - .collection(GENE_LISTS_COLLECTION) - .aggregate([ - { $match: { networkID: networkID.bson } }, - { $unwind: { path: "$genes" } }, - { $replaceRoot: { newRoot: "$genes" } }, - { $sort: { rank: -1 }} - ]); - return cursor; + async idExists(idStr) { + const id = makeID(idStr); + const result = await this.db + .collection(MOTIFS_AND_TRACKS_COLLECTION) + .count({ _id: id.bson }, { limit: 1 }); + return Boolean(result); } - async getGeneSetCollectionUsedByNetwork(networkIDString) { - const networkID = makeID(networkIDString); - const network = await this.db - .collection(NETWORKS_COLLECTION) - .findOne( - { _id: networkID.bson }, - { _id: 0, geneSetCollection: 1 } - ); - - return network.geneSetCollection; - } /** - * Returns an cursor of objects of the form: - * [ { "name": "My Gene Set", "description": "blah blah", "genes": ["ABC", "DEF"] }, ... ] + * Updates a document--only the 'name' can be updated. + * @returns true if the network has been found and updated, false otherwise. */ - async getGMTUsedByNetworkCursor(networkIDString) { - const networkID = makeID(networkIDString); - const geneSetCollection = await this.getGeneSetCollectionUsedByNetwork(networkIDString); - - const cursor = await this.db - .collection(NETWORKS_COLLECTION) - .aggregate([ - ...this._enrichmentQuery(networkID), - { $lookup: { - from: geneSetCollection, - localField: "name", - foreignField: "name", - as: "geneSet" - }}, - { $unwind: "$geneSet" }, - { $project: { - name: true, - description: "$geneSet.description", - genes: "$geneSet.genes", - }} - ]); + async updateState(idStr, { name }) { + const id = makeID(idStr); - return cursor; - } - - - /** - * Returns names - */ - async getNodeDataSetNames(networkIDString) { - const networkID = makeID(networkIDString); - - const names = await this.db - .collection(NETWORKS_COLLECTION) - .aggregate([ - // Get the node data in the network - { $match: { _id: networkID.bson } }, - { $replaceWith: { path: "$network.elements.nodes.data" } }, - { $unwind: { path: "$path" } }, - { $replaceRoot: { newRoot: "$path" } }, - // Get the names - { $unwind: { path: "$name" } }, - { $project: { name: 1 }} - ]) - .map(obj => obj.name) - .toArray(); - - return names; - } - - - /** - * Returns the entire gene/ranks document. - */ - async getRankedGeneList(networkIDString) { - const networkID = makeID(networkIDString); - const network = await this.db - .collection(GENE_LISTS_COLLECTION) - .findOne( - { networkID: networkID.bson }, - { projection: { _id: 0, min: 1, max: 1, genes: 1 } } + const res = await this.db + .collection(STATE_DATA_COLLECTION) + .updateOne( + { 'motifsAndTracksID': id.bson }, + { $set: { name: name } } ); - return network; - } - - - /** - * Returns the contents of a gene set, including the name, - * description and gene list. - */ - async getGeneSets(geneSetCollection, geneSetNames) { - return await this.db - .collection(geneSetCollection) - .find({ name: { $in: geneSetNames } }) - .project({ _id: 0 }) - .toArray(); - } - - - /** - * Returns the genes from one or more given gene sets joined with ranks. - * The returned array is sorted so that the genes with ranks are first (sorted by rank), - * then the genes without rankes are after (sorted alphabetically). - */ - async getGenesWithRanks(networkIDStr, geneSetNames, intersection) { - const networkID = makeID(networkIDStr); - const geneSetCollection = await this.getGeneSetCollectionUsedByNetwork(networkIDStr); - - if(geneSetNames === undefined || geneSetNames.length == 0) { - geneSetNames = await this.getNodeDataSetNames(networkID); - } - - const geneListWithRanks = await this.db - .collection(geneSetCollection) - .aggregate([ - { $match: { name: { $in: geneSetNames } } }, - { $project: { genes: { $map: { input: "$genes", as: "g", in: { gene: "$$g" } } } } }, - { $unwind: "$genes" }, - { $replaceRoot: { newRoot: "$genes" } }, - { $group: { _id: "$gene", gene: { $first: "$gene" }, count: { $count: {} } } }, - // if intersection=true then the the gene has to be in all the given genesets - ...(intersection - ? [{ $match: { $expr: { $eq: [ "$count", geneSetNames.length ] } } }] - : [] - ), - { $lookup: { - from: GENE_RANKS_COLLECTION, - let: { gene: "$gene" }, - pipeline: [ - { $match: - { $expr: - { $and: [ - { $eq: [ '$networkID', networkID.bson ] }, - { $eq: [ '$gene', '$$gene' ] } ] - } - } - } - ], - as: "newField" - } - }, - { $project: { _id: 0, gene: "$gene", rank: { $first: "$newField.rank" } } }, - { $match: { rank: { $exists: true } } }, - { $sort: { rank: -1, gene: 1 } } - ]) - .toArray(); - - return { - genes: geneListWithRanks - }; + return res.modifiedCount > 0; } - - async getGenesForSearchCursor(networkIDStr) { - // TODO: delete this -- prototype only - return this.RESULTS_CACHE.get(networkIDStr)?.genes; - //=========================================================== - - // const networkID = makeID(networkIDStr); - - // const cursor = await this.db - // .collection(GENE_RANKS_COLLECTION) - // .find( - // { networkID: networkID.bson }, - // { projection: { _id: 0, gene: 1 } } - // ); + // /** + // * { + // * _id: 'asdf', + // * networkID: "abcdefg", + // * positions: [ + // * { + // * id: "asdf-asdf-asdf", + // * x: 12.34 + // * y: 56.78 + // * collapsed: false + // * } + // * ] + // * } + // * Note: Deleted nodes are not part of the positions document. + // */ + // async setPositions(networkIDString, positions) { + // const networkID = makeID(networkIDString); + // const document = { + // networkID: networkID.bson, + // positions + // }; + + // // Save the document twice, the one in POSITIONS_RESTORE_COLLECTION never changes + // await this.db + // .collection(POSITIONS_RESTORE_COLLECTION) + // .updateOne({ networkID: networkID.bson }, { $setOnInsert: document }, { upsert: true }); + + // await this.db + // .collection(POSITIONS_COLLECTION) + // .replaceOne({ networkID: networkID.bson }, document, { upsert: true }); + // } + + // async getPositions(networkIDString) { + // const networkID = makeID(networkIDString); + + // let result = await this.db + // .collection(POSITIONS_COLLECTION) + // .findOne({ networkID: networkID.bson }); + + // if(!result) { + // console.log("Did not find positions, querying POSITIONS_RESTORE_COLLECTION"); + // result = await this.db + // .collection(POSITIONS_RESTORE_COLLECTION) + // .findOne({ networkID: networkID.bson }); + // } - // return cursor; - } + // return result; + // } + // async deletePositions(networkIDString) { + // const networkID = makeID(networkIDString); + // // Only delete from POSITIONS_COLLECTION, not from POSITIONS_RESTORE_COLLECTION + // await this.db + // .collection(POSITIONS_COLLECTION) + // .deleteOne({ networkID: networkID.bson } ); + // } - async getResultsForSearchCursor(networkIDStr) { - // TODO: delete this -- prototype only - return this.RESULTS_CACHE.get(networkIDStr)?.results; - //=========================================================== - - // const networkID = makeID(networkIDStr); - // const geneSetCollection = await this.getGeneSetCollectionUsedByNetwork(networkIDStr); - - // const cursor = await this.db - // .collection(NETWORKS_COLLECTION) - // .aggregate([ - // ...this._enrichmentQuery(networkID), - // { $lookup: { - // from: geneSetCollection, - // localField: "name", - // foreignField: "name", - // as: "geneSet" - // }}, - // { $project: { - // name: true, - // pval: true, - // padj: true, - // NES: true, - // size: true, - // mcode_cluster_id: true, - // genes: { $arrayElemAt: [ "$geneSet", 0 ] }, - // }}, - // { $project: { - // name: true, - // pval: true, - // padj: true, - // NES: true, - // size: true, - // mcode_cluster_id: true, - // description: "$genes.description", - // genes: "$genes.genes" - // }}, - // ]); - - // return cursor; - } - - async getNetworkCounts() { + async getResultCounts() { const result = await this.db - .collection(NETWORKS_COLLECTION) + .collection(MOTIFS_AND_TRACKS_COLLECTION) .aggregate([ { $facet: { 'user': [ @@ -727,27 +279,27 @@ class Datastore { return result[0]; } - - async getNetworkStatsCursor() { + async getResultStatsCursor() { const cursor = await this.db - .collection(NETWORKS_COLLECTION) + .collection(MOTIFS_AND_TRACKS_COLLECTION) .aggregate([ { $match: { demo: { $ne: true } }}, + { $lookup: { + from: 'stateData', + localField: '_id', + foreignField: 'motifsAndTracksID', + as: 'stateData' + }}, { $project: { - networkName: 1, + name: { $arrayElemAt: ["$stateData.name", 0] }, creationTime: 1, lastAccessTime: 1, - geneSetCollection: 1, - inputType: 1, - nodeCount: { $size: '$network.elements.nodes' }, - edgeCount: { $size: '$network.elements.edges' } }} ]); return cursor; } - } const ds = new Datastore(); // singleton diff --git a/src/server/env.js b/src/server/env.js index 9b92ecd3..4e48b7e1 100644 --- a/src/server/env.js +++ b/src/server/env.js @@ -27,7 +27,6 @@ export const TRACK_RANKINGS_DATABASE = process.env.TRACK_RANKINGS_DATABASE; // Mongo config export const MONGO_URL = process.env.MONGO_URL; export const MONGO_ROOT_NAME = process.env.MONGO_ROOT_NAME; -export const MONGO_COLLECTION_QUERIES = process.env.MONGO_COLLECTION_QUERIES; // Sentry config export const SENTRY_ENVIRONMENT = process.env.SENTRY_ENVIRONMENT; diff --git a/src/server/index.js b/src/server/index.js index 0c75286b..754ee015 100755 --- a/src/server/index.js +++ b/src/server/index.js @@ -28,7 +28,7 @@ import Datastore from './datastore.js'; // Connect to the database console.info('Starting Express'); -await Datastore.connect(); +await Datastore.initialize(); // Set up the debug log const debugLog = debug('iregulon'); // Set up the express app diff --git a/src/server/routes/api/create.js b/src/server/routes/api/create.js index 89517a77..81641458 100644 --- a/src/server/routes/api/create.js +++ b/src/server/routes/api/create.js @@ -1,10 +1,8 @@ import Express from 'express'; -import fs from 'fs/promises'; import * as Sentry from "@sentry/node"; import fetch from 'node-fetch'; import Datastore from '../../datastore.js'; import { annotateGenes, parseMotifsAndTracks } from '../../util.js'; -import { rankedGeneListToDocument, fgseaServiceGeneRanksToDocument } from '../../datastore.js'; import { performance } from 'perf_hooks'; import { IREGULON_JOB_SERVICE_URL, @@ -108,8 +106,6 @@ http.post('/', async function(req, res) { }); /* - * Runs the FGSEA/EnrichmentMap algorithms, saves the - * created network, then returns its ID. */ http.post('/demo', async function(req, res, next) { const perf = createPeformanceHook(); @@ -180,180 +176,6 @@ function createPeformanceHook() { }; } -// async function runDataPipeline({ networkName, contentType, type, classes, body, demo, perf }, res) { -// console.log('/api/create/'); -// // n.b. no await so as to not block -// saveUserUploadFileToS3(body, `${networkName}.csv`, contentType); - -// const preranked = type === 'preranked'; - -// perf.mark('bridgedb'); -// const needIdMapping = isEnsembl(body); -// if(needIdMapping) { -// body = await runEnsemblToHGNCMapping(body, contentType); -// } - -// perf.mark('fgsea'); -// let rankedGeneList; -// let pathwaysForEM; -// if(preranked) { -// const fgseaRes = await runFGSEApreranked(body, contentType); -// const { pathways, gmtFile } = fgseaRes; -// if(gmtFile !== GMT_FILE) { -// throw new CreateError({ step: 'fgsea', detail: 'gmt', message: `FGSEA: wrong GMT. Expected '${GMT_FILE}', got '${gmtFile}'.` }); -// } -// const delim = contentType === 'text/csv' ? ',' : '\t'; -// rankedGeneList = rankedGeneListToDocument(body, delim); -// pathwaysForEM = pathways; -// } else { -// // Messages from FGSEA are basically just warning about non-finite ranks -// const fgseaRes = await runFGSEArnaseq(body, classes, contentType); -// const { ranks, pathways, messages, gmtFile } = fgseaRes; -// if(gmtFile !== GMT_FILE) { -// throw new CreateError({ step: 'fgsea', detail: 'gmt', message: `FGSEA: wrong GMT. Expected '${GMT_FILE}', got '${gmtFile}'.` }); -// } -// sendMessagesToSentry('fgsea', messages); -// rankedGeneList = fgseaServiceGeneRanksToDocument(ranks); -// pathwaysForEM = pathways; -// } - -// perf.mark('em'); -// const networkJson = await runEM(pathwaysForEM, demo); -// if(isEmptyNetwork(networkJson)) { -// throw new CreateError({ step: 'em', detail: 'empty' }); -// } -// if(networkJson.gmtFile !== GMT_FILE) { -// throw new CreateError({ step: 'em', detail: 'gmt', message: `EM-Service: wrong GMT. Expected '${GMT_FILE}', got '${networkJson.gmtFile}'.` }); -// } - -// let networkID; -// try { -// perf.mark('mongo'); -// networkID = await Datastore.createNetwork(networkJson, networkName, type, GMT_FILE, demo); -// await Datastore.initializeGeneRanks(GMT_FILE, networkID, rankedGeneList); -// res?.send(networkID); -// } catch(e) { -// throw new CreateError({ step: 'mongo', cause: e }); -// } -// -// perf.mark('end'); -// -// Datastore.createPerfDocument(networkID, { -// startTime: perf.startTime, -// emptyNetwork: typeof networkID === 'undefined', -// geneCount: rankedGeneList?.genes?.length, -// steps: [ { -// step: 'bridgedb', -// needIdMapping, -// url: BRIDGEDB_URL, -// timeTaken: perf.measure({ from:'bridgedb', to:'fgsea' }), -// }, { -// step: 'fgsea', -// type, -// url: preranked ? FGSEA_PRERANKED_SERVICE_URL : FGSEA_RNASEQ_SERVICE_URL, -// timeTaken: perf.measure({ from:'fgsea', to:'em' }), -// }, { -// step: 'em', -// url: EM_SERVICE_URL, -// timeTaken: perf.measure({ from:'em', to:'mongo' }), -// }, { -// step: 'mongo', -// url: MONGO_URL, -// timeTaken: perf.measure({ from:'mongo', to:'end' }), -// }] -// }); - -// return networkID; -// } - - -function isEmptyNetwork(networkJson) { - return !(networkJson.network?.elements?.nodes?.length) - || !(networkJson.summaryNetwork?.elements?.nodes?.length); -} - - -// async function runFGSEApreranked(ranksData, contentType) { -// let response; -// try { -// response = await fetch(FGSEA_PRERANKED_SERVICE_URL, { -// method: 'POST', -// headers: { 'Content-Type': contentType }, -// body: ranksData -// }); -// } catch(e) { -// throw new CreateError({ step: 'fgsea', type: 'preranked', cause: e }); -// } -// if(!response.ok) { -// const body = await response.text(); -// const status = response.status; -// throw new CreateError({ step: 'fgsea', type: 'preranked', body, status }); -// } -// return await response.json(); -// } - - -// async function runFGSEArnaseq(countsData, classes, contentType) { -// const url = FGSEA_RNASEQ_SERVICE_URL + '?' + new URLSearchParams({ classes }); -// let response; -// try { -// response = await fetch(url, { -// method: 'POST', -// headers: { 'Content-Type': contentType }, -// body: countsData -// }); -// } catch(e) { -// throw new CreateError({ step: 'fgsea', type: 'rnaseq', cause: e }); -// } -// if(!response.ok) { -// const body = await response.text(); -// const status = response.status; -// throw new CreateError({ step: 'fgsea', type: 'rnaseq', body, status }); -// } -// return await response.json(); -// } - - -// async function runEM(fgseaResults, demo) { -// const body = { -// // We only support one dataSet -// dataSets: [{ -// name: "EM Web", -// method: "FGSEA", -// fgseaResults -// }], -// parameters: { -// // These parameters correspond to the fields in EMCreationParametersDTO -// // similarityMetric: "JACCARD", -// // similarityCutoff: 0.25, - -// // parameters only used by the demo network -// ...(demo && { -// qvalue: 0.0001, -// similarityMetric: "JACCARD", -// similarityCutoff: 0.5, -// }) -// } -// }; - -// let response; -// try { -// response = await fetch(EM_SERVICE_URL, { -// method: 'POST', -// headers: { 'Content-Type': 'application/json' }, -// body: JSON.stringify(body) -// }); -// } catch(e) { -// throw new CreateError({ step: 'em', cause: e }); -// } -// if(!response.ok) { -// const body = await response.text(); -// const status = response.status; -// throw new CreateError({ step: 'em', body, status }); -// } -// return await response.json(); -// } - /** * If the first gene is an ensembl ID then assume they all are. diff --git a/src/server/routes/api/export.js b/src/server/routes/api/export.js index f9278bcc..aaf3a27b 100644 --- a/src/server/routes/api/export.js +++ b/src/server/routes/api/export.js @@ -3,58 +3,7 @@ import Datastore from '../../datastore.js'; const http = Express.Router(); -// Return enrichment results in TSV format -http.get('/enrichment/:netid', async function(req, res, next) { - try { - const { netid } = req.params; - const cursor = await Datastore.getEnrichmentResultsCursor(netid); - - sendDataLines(cursor, res, { - header: 'pathway\tsize\tpval\tpadj\tNES', - objToStr: ({name, size, pval, padj, NES}) => `${name}\t${size}\t${pval}\t${padj}\t${NES}` - }); - - } catch(err) { - next(err); - } -}); - - -// Return ranked gene list in TSV format -http.get('/ranks/:netid', async function(req, res, next) { - try { - const { netid } = req.params; - const cursor = await Datastore.getRankedGeneListCursor(netid); - - sendDataLines(cursor, res, { - header: 'gene\trank', - objToStr: ({gene, rank}) => `${gene}\t${rank}` - }); - - } catch(err) { - next(err); - } -}); - - -// Return ranked gene list in TSV format -http.get('/gmt/:netid', async function(req, res, next) { - try { - const { netid } = req.params; - const cursor = await Datastore.getGMTUsedByNetworkCursor(netid); - - sendDataLines(cursor, res, { - header: 'name\tdescription\tgenes', - objToStr: ({name, description, genes}) => { - const genesStr = genes.join('\t'); - return `${name}\t${description}\t${genesStr}`; - } - }); - - } catch(err) { - next(err); - } -}); +// TODO async function sendDataLines(cursor, res, { type='tsv', header, objToStr } ) { diff --git a/src/server/routes/api/index.js b/src/server/routes/api/index.js index 9799ffad..7e4fc999 100644 --- a/src/server/routes/api/index.js +++ b/src/server/routes/api/index.js @@ -40,17 +40,6 @@ http.get('/sample-data', async function(req, res, next) { } }); -/* - * Returns the IDs of demo networks. - */ -http.get('/demos', async function(req, res, next) { - try { - const networkIDs = await Datastore.getDemoNetworkIDs(); - res.send(JSON.stringify(networkIDs)); - } catch (err) { - next(err); - } -}); /* * Returns a network given its ID. @@ -58,7 +47,7 @@ http.get('/demos', async function(req, res, next) { http.get('/:netid', async function(req, res, next) { try { const { netid } = req.params; - const network = await Datastore.getNetwork(netid); + const network = await Datastore.getMotifsAndTracks(netid); if (!network) { res.sendStatus(404); @@ -76,8 +65,9 @@ http.get('/:netid', async function(req, res, next) { http.put('/:netid', async function(req, res, next) { try { const { netid } = req.params; - const { networkName } = req.body; - const updated = await Datastore.updateNetwork(netid, { networkName }); + const { name } = req.body; + console.log('updateState', 'networkName:', name); + const updated = await Datastore.updateState(netid, { name }); res.sendStatus(updated ? 204 : 409); } catch (err) { @@ -86,66 +76,14 @@ http.put('/:netid', async function(req, res, next) { }); -/* - * Returns a ranked gene list. - */ -http.get('/:netid/ranks', async function(req, res, next) { - try { - const { netid } = req.params; - - const rankedGeneList = await Datastore.getRankedGeneList(netid); - if(!rankedGeneList) { - res.sendStatus(404); - } else { - res.send(JSON.stringify(rankedGeneList)); - } - } catch (err) { - next(err); - } -}); - - -/* - * Returns the contents of multiple gene sets, including ranks. - * Can be used to populate the gene search documents on the clinent. - */ -http.post('/:netid/genesets', async function(req, res, next) { - try { - const { intersection } = req.query; - const { netid } = req.params; - const { geneSets } = req.body; - - if(!Array.isArray(geneSets)) { - res.sendStatus(404); - return; - } - - const geneInfo = await Datastore.getGenesWithRanks(netid, geneSets, intersection === 'true'); - if(!geneInfo) { - res.sendStatus(404); - } else { - res.send(JSON.stringify(geneInfo)); - } - } catch (err) { - next(err); - } -}); - - /* * Returns the all the genes and ranks in the given network. */ http.get('/:netid/genesforsearch', async function(req, res, next) { try { const { netid } = req.params; - - // TODO - this is temporary for prototyping - const genes = await Datastore.getGenesForSearchCursor(netid); + const genes = await Datastore.getGenesForSearch(netid); res.write(JSON.stringify(genes)); - // ============================================================================================================ - // const cursor = await Datastore.getGenesForSearchCursor(netid); - // await writeCursorToResult(cursor, res); - // cursor.close(); } catch (err) { next(err); } finally { @@ -160,15 +98,8 @@ http.get('/:netid/genesforsearch', async function(req, res, next) { http.get('/:netid/results', async function(req, res, next) { try { const { netid } = req.params; - - // TODO - this is temporary for prototyping - const results = await Datastore.getResultsForSearchCursor(netid); + const results = await Datastore.getResultsForSearch(netid); res.write(JSON.stringify(results)); - // ============================================================================================================ - // const cursor = await Datastore.getResultsForSearchCursor(netid); - // await writeCursorToResult(cursor, res); - // cursor.close(); - } catch (err) { next(err); } finally { @@ -177,51 +108,51 @@ http.get('/:netid/results', async function(req, res, next) { }); -http.get('/:netid/positions', async function(req, res, next) { - try { - const { netid } = req.params; +// http.get('/:netid/positions', async function(req, res, next) { +// try { +// const { netid } = req.params; - const positions = await Datastore.getPositions(netid); - if(!positions) { - res.sendStatus(404); - } else { - res.send(JSON.stringify(positions)); - } +// const positions = await Datastore.getPositions(netid); +// if(!positions) { +// res.sendStatus(404); +// } else { +// res.send(JSON.stringify(positions)); +// } - res.sendStatus(404); +// res.sendStatus(404); - } catch (err) { - next(err); - } -}); - -http.post('/:netid/positions', async function(req, res, next) { - try { - const { netid } = req.params; - const { positions } = req.body; - - if(!Array.isArray(positions)) { - res.sendStatus(404); - return; - } - - await Datastore.setPositions(netid, positions); - - res.send('OK'); - } catch (err) { - next(err); - } -}); - -http.delete('/:netid/positions', async function(req, res, next) { - try { - const { netid } = req.params; - await Datastore.deletePositions(netid); - res.send('OK'); - } catch (err) { - next(err); - } -}); +// } catch (err) { +// next(err); +// } +// }); + +// http.post('/:netid/positions', async function(req, res, next) { +// try { +// const { netid } = req.params; +// const { positions } = req.body; + +// if(!Array.isArray(positions)) { +// res.sendStatus(404); +// return; +// } + +// await Datastore.setPositions(netid, positions); + +// res.send('OK'); +// } catch (err) { +// next(err); +// } +// }); + +// http.delete('/:netid/positions', async function(req, res, next) { +// try { +// const { netid } = req.params; +// await Datastore.deletePositions(netid); +// res.send('OK'); +// } catch (err) { +// next(err); +// } +// }); export async function writeCursorToResult(cursor, res) { diff --git a/src/server/routes/api/report.js b/src/server/routes/api/report.js index 073aed47..55fa0169 100644 --- a/src/server/routes/api/report.js +++ b/src/server/routes/api/report.js @@ -15,7 +15,7 @@ http.get(`/count/:secret`, async function(req, res, next) { return; } - const counts = await Datastore.getNetworkCounts(); + const counts = await Datastore.getResultCounts(); res.send(JSON.stringify(counts)); } catch(err) { @@ -32,7 +32,7 @@ http.get(`/networks/:secret`, async function(req, res, next) { return; } - const cursor = await Datastore.getNetworkStatsCursor(); + const cursor = await Datastore.getResultStatsCursor(); await writeCursorToResult(cursor, res); cursor.close();