🚧 Add venn route (WIP) (#234)

* :contruction: Add venn route (WIP)

* 📝 Make a check on sqons length before processing

package.json

@@ -37,12 +37,13 @@
   "homepage": "https://github.com/kids-first/kf-api-arranger#readme",
   "dependencies": {
     "@arranger/admin": "^2.19.4",
-    "@arranger/middleware": "^2.19.4",
     "@arranger/mapping-utils": "^2.19.4",
+    "@arranger/middleware": "^2.19.4",
     "@arranger/server": "^2.19.4",
     "@aws-sdk/client-s3": "^3.689.0",
     "@aws-sdk/s3-request-presigner": "^3.689.0",
     "@elastic/elasticsearch": "^7.11.0",
+    "@overture-stack/sqon-builder": "^1.1.0",
     "ajv": "^8.17.1",
     "cors": "^2.8.5",
     "dotenv": "^16.4.5",

src/app.ts

@@ -19,6 +19,7 @@ import { CreateSetBody, Set, SetSqon, UpdateSetContentBody, UpdateSetTagBody } f
 import { getStatistics, getStudiesStatistics } from './endpoints/statistics';
 import transcriptomicsRouter from './endpoints/transcriptomics/route';
 import { computeUpset } from './endpoints/upset';
+import { venn } from './endpoints/venn/venn';
 import { esHost, keycloakURL, userApiURL } from './env';
 import { globalErrorHandler, globalErrorLogger } from './errors';
 import {
@@ -30,6 +31,9 @@ import {
 } from './middleware/cache';
 import { injectBodyHttpHeaders } from './middleware/injectBodyHttpHeaders';
 import { resolveSetIdMiddleware } from './middleware/resolveSetIdInSqon';
+import { sqonContainsSet, renameFieldToFieldName } from './sqon/manipulateSqon';
+import { resolveSetsInSqon } from './sqon/resolveSetInSqon';
+import { StatusCodes } from 'http-status-codes';
 
 export default (keycloak: Keycloak, getProject: (projectId: string) => ArrangerProject): Express => {
     const app = express();
@@ -206,6 +210,32 @@ export default (keycloak: Keycloak, getProject: (projectId: string) => ArrangerP
         }
     });
 
+    app.post('/venn', keycloak.protect(), async (req, res, next) => {
+        try {
+            if ([2, 3].includes(req.body?.sqons?.length)) {
+                // Convert sqon(s) with set_id if exists to intelligible sqon for ES query translation.
+                const sqons: string[] = [];
+                for (const s of req.body.sqons) {
+                    if (sqonContainsSet(s)) {
+                        const accessToken = req.headers.authorization;
+                        const r = await resolveSetsInSqon(s, null, accessToken);
+                        sqons.push(JSON.stringify(r));
+                    } else {
+                        sqons.push(s);
+                    }
+                }
+                const data = await venn(sqons);
+                res.send({
+                    data,
+                });
+            } else {
+                res.status(StatusCodes.UNPROCESSABLE_ENTITY).send('Bad Inputs');
+            }
+        } catch (e) {
+            next(e);
+        }
+    });
+
     app.use(globalErrorLogger, globalErrorHandler);
 
     return app;

src/endpoints/upset.ts

@@ -1,7 +1,7 @@
-import { getNestedFields } from '@arranger/mapping-utils';
 import { buildQuery } from '@arranger/middleware';
 
 import EsInstance from '../ElasticSearchClientInstance';
+import { getNestedFieldsForIndex } from '../sqon/getNestedFieldsForIndex';
 
 export type UpsetData = {
     name: string;
@@ -60,8 +60,7 @@ export const computeUpset = async (
 
     const needToFetchMapping = !nestedFields || nestedFields.length === 0;
     if (needToFetchMapping) {
-        const rM = await client.indices.getMapping({ index: 'participant_centric' });
-        nestedFields = getNestedFields(Object.values(rM.body || {})[0]?.mappings?.properties);
+        nestedFields = await getNestedFieldsForIndex(client, 'participant_centric');
     }
 
     // assumption: unique participants

src/endpoints/venn/venn.ts

@@ -0,0 +1,141 @@
+import { buildQuery } from '@arranger/middleware';
+import { default as SQONBuilder, SQON as v3SQON } from '@overture-stack/sqon-builder';
+
+import EsInstance from '../../ElasticSearchClientInstance';
+import { getNestedFieldsForIndex } from '../../sqon/getNestedFieldsForIndex';
+import { renameFieldNameToField, renameFieldToFieldName } from '../../sqon/manipulateSqon';
+
+type OutputElement = {
+    operation: string;
+    count: number;
+};
+
+type Output = {
+    basics: OutputElement[];
+    alloys: OutputElement[];
+};
+
+const builder = SQONBuilder;
+
+const setFormulasDuo = (s1: v3SQON, s2: v3SQON) => [
+    {
+        operation: 'Q₁',
+        count: undefined,
+        sqon: builder.from(s1),
+    },
+    {
+        operation: 'Q₂',
+        count: undefined,
+        sqon: builder.from(s2),
+    },
+    {
+        operation: '(Q₁∩Q₂)',
+        sqon: builder.and([s1, s2]),
+    },
+    {
+        operation: '(Q₁)-(Q₂)',
+        sqon: builder.from(s1).not(builder.from(s2)),
+    },
+    {
+        operation: '(Q₂)-(Q₁)',
+        sqon: builder.from(s2).not(builder.from(s1)),
+    },
+];
+
+const setFormulasTrio = (s1: v3SQON, s2: v3SQON, s3: v3SQON) =>
+    [
+        {
+            operation: 'Q₁',
+            sqon: builder.from(s1),
+        },
+        {
+            operation: 'Q₂',
+            sqon: builder.from(s2),
+        },
+        {
+            operation: 'Q₃',
+            sqon: builder.from(s3),
+        },
+        {
+            operation: '(Q₁∩Q₂∩Q₃)',
+            sqon: builder.and([s1, s2, s3]),
+        },
+        {
+            operation: '(Q₁∩Q₂)-(Q₃)',
+            sqon: builder.and([s1, s2]).not(s3),
+        },
+        {
+            operation: '(Q₂∩Q₃)-(Q₁)',
+            sqon: builder.and([s2, s3]).not(s1),
+        },
+        {
+            operation: '(Q₁∩Q₃)-(Q₂)',
+            sqon: builder.and([s1, s3]).not(s2),
+        },
+        {
+            operation: '(Q₁)-(Q₂∩Q₃)',
+            sqon: builder.from(s1).not(builder.or([s2, s3])),
+        },
+        {
+            operation: '(Q₂)-(Q₁∩Q₃)',
+            sqon: builder.from(s2).not(builder.or([s1, s3])),
+        },
+        {
+            operation: '(Q₃)-(Q₁∩Q₂)',
+            sqon: builder.from(s3).not(builder.or([s1, s2])),
+        },
+    ].map(x => ({ ...x, sqon: renameFieldNameToField(x.sqon) }));
+
+let nestedFields: string[] = null;
+
+export const venn = async (sqons: string[]): Promise<Output> => {
+    // (Arranger v2 vs v3): SqonBuilder uses property "fieldName" (v3) but we use "field" (v2)
+    const v3Sqons: v3SQON[] = sqons.map(s => renameFieldToFieldName(s));
+
+    const setFormulas =
+        sqons.length === 2
+            ? setFormulasDuo(v3Sqons[0], v3Sqons[1])
+            : setFormulasTrio(v3Sqons[0], v3Sqons[1], v3Sqons[2]);
+
+    const client = EsInstance.getInstance();
+    const needToFetchMapping = !nestedFields || nestedFields.length === 0;
+    if (needToFetchMapping) {
+        nestedFields = await getNestedFieldsForIndex(client, 'participant_centric');
+    }
+
+    const mSearchBody = setFormulas
+        .map(x => [
+            {},
+            {
+                track_total_hits: true,
+                size: 0,
+                query: buildQuery({
+                    nestedFields: nestedFields,
+                    filters: x.sqon,
+                }),
+            },
+        ])
+        .flat();
+
+    const r = await client.msearch({
+        body: mSearchBody,
+    });
+
+    const responses = r.body?.responses || [];
+
+    const data = setFormulas.map((x, i) => ({
+        ...x,
+        count: responses[i].hits.total.value,
+    }));
+
+    // Reformat for UI
+    return data.reduce(
+        (xs: Output, x: OutputElement) => {
+            if (['Q₁', 'Q₂', 'Q₃'].some(y => y === x.operation)) {
+                return { ...xs, basics: [...xs.basics, x] };
+            }
+            return { ...xs, alloys: [...xs.alloys, x] };
+        },
+        { basics: [], alloys: [] },
+    );
+};

src/sqon/getNestedFieldsForIndex.ts

@@ -0,0 +1,7 @@
+import { getNestedFields } from '@arranger/mapping-utils';
+import { Client } from '@elastic/elasticsearch';
+
+export const getNestedFieldsForIndex = async (client: Client, indexName: string): Promise<string[]> => {
+    const rM = await client.indices.getMapping({ index: indexName });
+    return getNestedFields(Object.values(rM.body || {})[0]?.mappings?.properties);
+};

src/sqon/manipulateSqon.ts

@@ -1,3 +1,5 @@
+import { SQON as v3SQON } from '@overture-stack/sqon-builder';
+
 import { SetSqon } from '../endpoints/sets/setsTypes';
 
 export const addSqonToSetSqon = (receivingSqon: SetSqon, donorSqon: SetSqon): SetSqon =>
@@ -16,3 +18,17 @@ export const removeSqonToSetSqon = (setSqon: SetSqon, sqonToRemove: SetSqon): Se
         content: [setSqon, negatedSqonToRemove],
     } as SetSqon;
 };
+
+const isString = (x: unknown): boolean => typeof x === 'string';
+
+const renameFieldCore = (toV3: boolean) => (sqon: v3SQON | string) => {
+    const s = (isString(sqon) ? sqon : JSON.stringify(sqon)) as string;
+    const updated = toV3 ? s.replace(/"field":/g, '"fieldName":') : s.replace(/"fieldName":/g, '"field":');
+    return JSON.parse(updated);
+};
+
+export const renameFieldNameToField = renameFieldCore(false);
+
+export const renameFieldToFieldName = renameFieldCore(true);
+
+export const sqonContainsSet = (s: string) => JSON.stringify(s).includes('"set_id:');

src/sqon/types.ts

@@ -0,0 +1,6 @@
+//A poor' s man sqon type.
+export type Sqon = {
+    op: string;
+    content: any;
+    [key: string]: any;
+};