🔧 Support setId translations in sqons for the Venn route

Author: evans-g-crsj

src/app.ts

@@ -20,20 +20,13 @@ import { CreateSetBody, Set, SetSqon, UpdateSetContentBody, UpdateSetTagBody } f
 import { getStatistics, getStudiesStatistics } from './endpoints/statistics';
 import transcriptomicsRouter from './endpoints/transcriptomics/route';
 import { computeUpset } from './endpoints/upset';
-import { venn } from './endpoints/venn/venn';
+import { reformatVenn, venn } from './endpoints/venn/venn';
 import { esHost, keycloakURL, userApiURL } from './env';
 import { globalErrorHandler, globalErrorLogger } from './errors';
-import {
-    flushAllCache,
-    STATISTICS_CACHE_ID,
-    STATISTICS_PUBLIC_CACHE_ID,
-    twineWithCache,
-} from './middleware/cache';
+import { flushAllCache, STATISTICS_CACHE_ID, STATISTICS_PUBLIC_CACHE_ID, twineWithCache } from './middleware/cache';
 import { injectBodyHttpHeaders } from './middleware/injectBodyHttpHeaders';
 import { resolveSetIdMiddleware } from './middleware/resolveSetIdInSqon';
-import { sqonContainsSet } from './sqon/manipulateSqon';
-import { resolveSetsInSqon } from './sqon/resolveSetInSqon';
-import { Sqon } from './sqon/types';
+import { replaceIdsWithSetId, resolveSetsInAllSqonsWithMapper } from './sqon/resolveSetInSqon';
 
 export default (keycloak: Keycloak, getProject: (projectId: string) => ArrangerProject): Express => {
     const app = express();
@@ -196,7 +189,7 @@ export default (keycloak: Keycloak, getProject: (projectId: string) => ArrangerP
     });
 
     app.post('/authorized-studies', keycloak.protect(), async (req, res, next) => {
-        computeAuthorizedStudiesForAllFences(req, res, next);
+        await computeAuthorizedStudiesForAllFences(req, res, next);
     });
 
     app.post('/upset', keycloak.protect(), async (req, res, next) => {
@@ -210,25 +203,21 @@ export default (keycloak: Keycloak, getProject: (projectId: string) => ArrangerP
 
     app.post('/venn', keycloak.protect(), async (req, res, next) => {
         try {
-            if ([2, 3].includes(req.body?.sqons?.length)) {
-                // Convert sqon(s) with set_id if exists to intelligible sqon for ES query translation.
-                const sqons: Sqon[] = [];
-                for (const s of req.body.sqons) {
-                    if (sqonContainsSet(s)) {
-                        const accessToken = req.headers.authorization;
-                        const r = await resolveSetsInSqon(s, null, accessToken);
-                        sqons.push(r);
-                    } else {
-                        sqons.push(s);
-                    }
-                }
-                const data = await venn(sqons);
-                res.send({
-                    data,
-                });
-            } else {
+            if (![2, 3].includes(req.body?.sqons?.length)) {
                 res.status(StatusCodes.UNPROCESSABLE_ENTITY).send('Bad Inputs');
+                return;
             }
+            // Convert sqon(s) with set_id if exists to intelligible sqon for ES query translation.
+            const { resolvedSqons: sqons, m: mSetItToIds } = await resolveSetsInAllSqonsWithMapper(
+                req.body.sqons,
+                null,
+                req.headers.authorization,
+            );
+            const data1 = await venn(sqons);
+            const data2 = data1.map(x => ({ ...x, sqon: replaceIdsWithSetId(x.sqon, mSetItToIds) }));
+            res.send({
+                data: reformatVenn(data2),
+            });
         } catch (e) {
             next(e);
         }

src/endpoints/venn/venn.ts

@@ -5,15 +5,19 @@ import { getNestedFieldsForIndex } from '../../sqon/getNestedFieldsForIndex';
 import { and, not } from '../../sqon/manipulateSqon';
 import { Sqon } from '../../sqon/types';
 
-type OutputElement = {
+type Output = {
     operation: string;
     count: number;
+    sqon: Sqon;
+};
+
+type OutputReformattedElement = Output & {
     setId?: string;
 };
 
-type Output = {
-    summary: OutputElement[];
-    operations: OutputElement[];
+type OutputReformatted = {
+    summary: OutputReformattedElement[];
+    operations: OutputReformattedElement[];
 };
 
 const setFormulasDuo = (s1: Sqon, s2: Sqon) => [
@@ -84,7 +88,7 @@ const setFormulasTrio = (s1: Sqon, s2: Sqon, s3: Sqon) => [
 
 let nestedFields: string[] = null;
 
-export const venn = async (sqons: Sqon[]): Promise<Output> => {
+export const venn = async (sqons: Sqon[]): Promise<Output[]> => {
     const setFormulas =
         sqons.length === 2 ? setFormulasDuo(sqons[0], sqons[1]) : setFormulasTrio(sqons[0], sqons[1], sqons[2]);
 
@@ -114,14 +118,15 @@ export const venn = async (sqons: Sqon[]): Promise<Output> => {
 
     const responses = r.body?.responses || [];
 
-    const data = setFormulas.map((x, i) => ({
+    return setFormulas.map((x, i) => ({
         ...x,
         count: responses[i].hits.total.value,
     }));
+};
 
-    // Reformatting for UI
+export const reformatVenn = (data: Output[]): OutputReformatted => {
     const tables = data.reduce(
-        (xs: Output, x: OutputElement) => {
+        (xs: OutputReformatted, x: OutputReformattedElement) => {
             if (['Q₁', 'Q₂', 'Q₃'].some(y => y === x.operation)) {
                 return { ...xs, summary: [...xs.summary, x] };
             }
@@ -132,6 +137,6 @@ export const venn = async (sqons: Sqon[]): Promise<Output> => {
 
     return {
         summary: tables.summary,
-        operations: tables.operations.map((x: OutputElement, i: number) => ({ ...x, setId: `set-${i}` })),
+        operations: tables.operations.map((x: Output, i: number) => ({ ...x, setId: `set-${i}` })),
     };
 };

src/sqon/manipulateSqon.ts

@@ -18,7 +18,7 @@ export const removeSqonToSetSqon = (setSqon: SetSqon, sqonToRemove: SetSqon): Se
     } as SetSqon;
 };
 
-export const sqonContainsSet = (s: string) => JSON.stringify(s).includes('"set_id:');
+export const sqonContainsSet = (s: Sqon) => JSON.stringify(s).includes('"set_id:');
 
 // Taken as-is from "@overture-stack/sqon-builder": "^1.1.0" but zod Types were removed
 const combine = (op, sqon, content, pivot) => {

src/sqon/resolveSetInSqon.ts

@@ -2,6 +2,8 @@ import { Dictionary, flattenDeep, get, isArray, zipObject } from 'lodash';
 
 import { SetSqon } from '../endpoints/sets/setsTypes';
 import { getSharedSet, getUserSets, UserSet } from '../userApi/userApiClient';
+import { sqonContainsSet } from './manipulateSqon';
+import { Sqon } from './types';
 
 const getSetIdsFromSqon = (sqon: SetSqon, collection = []) =>
     (isArray(sqon.content)
@@ -28,7 +30,14 @@ const injectIdsIntoSqon = (sqon: SetSqon, setIdsToValueMap: Dictionary<string[]>
     })),
 });
 
-export const resolveSetsInSqon = async (sqon: SetSqon, userId: string, accessToken: string): Promise<SetSqon> => {
+export const resolveSetsInSqonWithMapper = async (
+    sqon: SetSqon,
+    userId: string,
+    accessToken: string,
+): Promise<{
+    resolvedSqon: SetSqon;
+    m?: Dictionary<string[]>;
+}> => {
     const setIds: string[] = getSetIdsFromSqon(sqon || ({} as SetSqon));
     if (setIds.length) {
         const userSets: UserSet[] = await retrieveSetsFromUsers(accessToken, setIds);
@@ -38,12 +47,47 @@ export const resolveSetsInSqon = async (sqon: SetSqon, userId: string, accessTok
             ids,
         );
 
-        return injectIdsIntoSqon(sqon, setIdsToValueMap);
-    } else {
-        return sqon;
+        return {
+            resolvedSqon: injectIdsIntoSqon(sqon, setIdsToValueMap),
+            m: setIdsToValueMap,
+        };
     }
+    return {
+        resolvedSqon: sqon,
+        m: null,
+    };
 };
 
+export const resolveSetsInAllSqonsWithMapper = async (
+    sqons: Sqon[],
+    userId: string,
+    accessToken: string,
+): Promise<{
+    resolvedSqons: Sqon[];
+    m?: Map<string, string[]>;
+}> => {
+    const resolvedSqons = [];
+    let mSetItToIds = new Map();
+    for (const s of sqons) {
+        if (sqonContainsSet(s)) {
+            const r = await resolveSetsInSqonWithMapper(s, null, accessToken);
+            resolvedSqons.push(r.resolvedSqon);
+            if (r.m) {
+                mSetItToIds = new Map([...mSetItToIds, ...new Map(Object.entries(r.m))]);
+            }
+        } else {
+            resolvedSqons.push(s);
+        }
+    }
+    return {
+        resolvedSqons: resolvedSqons,
+        m: mSetItToIds,
+    };
+};
+
+export const resolveSetsInSqon = async (sqon: SetSqon, userId: string, accessToken: string): Promise<SetSqon> =>
+    (await resolveSetsInSqonWithMapper(sqon, userId, accessToken)).resolvedSqon;
+
 export const retrieveSetsFromUsers = async (accessToken: string, setIds: string[]): Promise<UserSet[]> => {
     // Get all user sets
     const userSets = await getUserSets(accessToken);
@@ -60,3 +104,28 @@ export const retrieveSetsFromUsers = async (accessToken: string, setIds: string[
 
     return userSets;
 };
+
+const hasSameElements = (a, b) => a.length === b.length && [...new Set(a)].every(ax => b.includes(ax));
+
+export const replaceIdsWithSetId = (sqon: Sqon, setIdsToValueMap: Map<string, string[]>): Sqon => ({
+    ...sqon,
+    content: sqon.content.map(x => {
+        if (Array.isArray(x.content)) {
+            return {
+                ...x,
+                content: replaceIdsWithSetId(x, setIdsToValueMap).content,
+            };
+        }
+
+        const setId = Array.isArray(x.content.value)
+            ? [...setIdsToValueMap].find(([, v]) => hasSameElements(v, x.content.value))?.[0]
+            : null;
+        return {
+            ...x,
+            content: {
+                ...x.content,
+                value: setId ? [setId] : x.content.value,
+            },
+        };
+    }),
+});