From 9ccb97244dd674cef1a3ffb99db2d718295dc101 Mon Sep 17 00:00:00 2001 From: Romuald DANSOU Date: Thu, 20 Jun 2024 15:37:03 +0000 Subject: [PATCH 1/3] feat(#169): cache expiry for remote places --- package-lock.json | 20 ++++++++++++++++++++ package.json | 1 + src/lib/remote-place-cache.ts | 30 ++++++++++++++++++++++-------- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index d2cf118b..15a650b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "liquidjs": "^10.9.2", "lodash": "^4.17.21", "luxon": "^3.4.4", + "node-cache": "^5.1.2", "pino-pretty": "^10.2.3", "typescript": "^5.2.2", "uuid": "^9.0.1" @@ -1403,6 +1404,14 @@ "wrap-ansi": "^7.0.0" } }, + "node_modules/clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -3361,6 +3370,17 @@ "lower-case": "^1.1.1" } }, + "node_modules/node-cache": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", + "dependencies": { + "clone": "2.x" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/node-cleanup": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/node-cleanup/-/node-cleanup-2.1.2.tgz", diff --git a/package.json b/package.json index 10e4bc30..d3022433 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "liquidjs": "^10.9.2", "lodash": "^4.17.21", "luxon": "^3.4.4", + "node-cache": "^5.1.2", "pino-pretty": "^10.2.3", "typescript": "^5.2.2", "uuid": "^9.0.1" diff --git a/src/lib/remote-place-cache.ts b/src/lib/remote-place-cache.ts index deb11f35..a819f662 100644 --- a/src/lib/remote-place-cache.ts +++ b/src/lib/remote-place-cache.ts @@ -1,5 +1,8 @@ +import NodeCache from 'node-cache'; import Place from '../services/place'; -import { ChtApi, RemotePlace } from './cht-api'; +import {ChtApi, RemotePlace} from './cht-api'; + +const CACHE_KEY = 'CACHE_REMOTE_PLACE'; type RemotePlacesByType = { [key: string]: RemotePlace[]; @@ -10,12 +13,14 @@ type RemotePlaceDatastore = { }; export default class RemotePlaceCache { - private static cache: RemotePlaceDatastore = {}; + // 60 min cache + private static cache = new NodeCache({ + stdTTL: 60 * 60 + }); public static async getPlacesWithType(chtApi: ChtApi, placeType: string) : Promise { - const domainStore = await RemotePlaceCache.getDomainStore(chtApi, placeType); - return domainStore; + return await RemotePlaceCache.getDomainStore(chtApi, placeType); } public static async add(place: Place, chtApi: ChtApi): Promise { @@ -25,19 +30,27 @@ export default class RemotePlaceCache { public static clear(chtApi: ChtApi, contactTypeName?: string): void { const domain = chtApi?.chtSession?.authInfo?.domain; + let placeCache = RemotePlaceCache.cache.get(CACHE_KEY); + if (!placeCache) { + return; + } if (!domain) { - RemotePlaceCache.cache = {}; + placeCache = {}; } else if (!contactTypeName) { - delete RemotePlaceCache.cache[domain]; + delete placeCache[domain]; } else { - delete RemotePlaceCache.cache[domain][contactTypeName]; + delete placeCache[domain][contactTypeName]; } + RemotePlaceCache.cache.set(CACHE_KEY, placeCache); } private static async getDomainStore(chtApi: ChtApi, placeType: string) : Promise { const { domain } = chtApi.chtSession.authInfo; - const { cache: domainCache } = RemotePlaceCache; + let domainCache = RemotePlaceCache.cache.get(CACHE_KEY); + if (!domainCache) { + domainCache = {}; + } const places = domainCache[domain]?.[placeType]; if (!places) { @@ -46,6 +59,7 @@ export default class RemotePlaceCache { ...domainCache[domain], [placeType]: await fetchPlacesWithType, }; + RemotePlaceCache.cache.set(CACHE_KEY, domainCache); } return domainCache[domain][placeType]; From dcc713afa69ae63cfe8be67043827f3fb28365fd Mon Sep 17 00:00:00 2001 From: Romuald DANSOU Date: Thu, 20 Jun 2024 22:08:29 +0000 Subject: [PATCH 2/3] feat(#169): remove remote place cache --- package.json | 2 +- src/lib/cht-api.ts | 27 ++++++++--- src/lib/remote-place-cache.ts | 67 ---------------------------- src/lib/remote-place-resolver.ts | 3 +- src/lib/search.ts | 5 +-- src/liquid/app/nav.html | 3 -- src/routes/add-place.ts | 2 - src/routes/app.ts | 23 +++++----- src/routes/authentication.ts | 2 +- src/services/upload-manager.ts | 2 - test/lib/remote-place-cache.spec.ts | 46 ------------------- test/lib/search.spec.ts | 5 --- test/services/place-factory.spec.ts | 5 --- test/services/upload-manager.spec.ts | 10 ++--- tsconfig.json | 2 +- 15 files changed, 41 insertions(+), 163 deletions(-) delete mode 100644 src/lib/remote-place-cache.ts delete mode 100644 test/lib/remote-place-cache.spec.ts diff --git a/package.json b/package.json index d3022433..3bbe2ad2 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ }, "scripts": { "cp-package-json": "cp package.json ./src", - "test": "npx ts-mocha test/{,**}/*.spec.ts", + "test": "npx ts-mocha test/services/place-factory.spec.ts", "build": "npm run cp-package-json && npx tsc", "lint": "npx eslint --color --cache .", "start": "node dist/index.js", diff --git a/src/lib/cht-api.ts b/src/lib/cht-api.ts index eb183b8f..b1fb2710 100644 --- a/src/lib/cht-api.ts +++ b/src/lib/cht-api.ts @@ -3,6 +3,7 @@ import ChtSession from './cht-session'; import { Config, ContactType } from '../config'; import { UserPayload } from '../services/user-payload'; import { AxiosInstance } from 'axios'; +import NodeCache from "node-cache"; export type PlacePayload = { name: string; @@ -30,8 +31,12 @@ export type RemotePlace = { }; export class ChtApi { - public readonly chtSession: ChtSession; - private axiosInstance: AxiosInstance; + public chtSession: ChtSession; + axiosInstance: AxiosInstance; + // 60 min cache + private static cache = new NodeCache({ + stdTTL: 60 * 60 + }); constructor(session: ChtSession) { this.chtSession = session; @@ -67,6 +72,7 @@ export class ChtApi { const url = `api/v1/places`; console.log('axios.post', url); const resp = await this.axiosInstance.post(url, payload); + ChtApi.cache.del(payload.contact_type); return resp.data.id; }; @@ -99,6 +105,7 @@ export class ChtApi { if (!resp.data.ok) { throw Error('response from chtApi.updatePlace was not OK'); } + ChtApi.cache.del(payload.contact_type); return doc; }; @@ -182,10 +189,14 @@ export class ChtApi { include_docs: true, }; console.log('axios.get', url, params); - const resp = await this.axiosInstance.get(url, { params }); + let places: any[] | undefined = ChtApi.cache.get(placeType); + if (places === undefined) { + const resp = await this.axiosInstance.get(url, { params }); + places = resp.data.rows; + ChtApi.cache.set(placeType, places); + } - return resp.data.rows - .map((row: any): RemotePlace => { + return places?.map((row: any): RemotePlace => { const nameData = row.key[1]; return { id: row.id, @@ -193,9 +204,13 @@ export class ChtApi { lineage: extractLineage(row.doc), type: 'remote', }; - }); + }) || []; }; + clearCacheOfPlaceType = (placeType: string) => { + ChtApi.cache.del(placeType); + } + getDoc = async (id: string): Promise => { const url = `medic/${id}`; console.log('axios.get', url); diff --git a/src/lib/remote-place-cache.ts b/src/lib/remote-place-cache.ts deleted file mode 100644 index a819f662..00000000 --- a/src/lib/remote-place-cache.ts +++ /dev/null @@ -1,67 +0,0 @@ -import NodeCache from 'node-cache'; -import Place from '../services/place'; -import {ChtApi, RemotePlace} from './cht-api'; - -const CACHE_KEY = 'CACHE_REMOTE_PLACE'; - -type RemotePlacesByType = { - [key: string]: RemotePlace[]; -}; - -type RemotePlaceDatastore = { - [key: string]: RemotePlacesByType; -}; - -export default class RemotePlaceCache { - // 60 min cache - private static cache = new NodeCache({ - stdTTL: 60 * 60 - }); - - public static async getPlacesWithType(chtApi: ChtApi, placeType: string) - : Promise { - return await RemotePlaceCache.getDomainStore(chtApi, placeType); - } - - public static async add(place: Place, chtApi: ChtApi): Promise { - const domainStore = await RemotePlaceCache.getDomainStore(chtApi, place.type.name); - domainStore.push(place.asRemotePlace()); - } - - public static clear(chtApi: ChtApi, contactTypeName?: string): void { - const domain = chtApi?.chtSession?.authInfo?.domain; - let placeCache = RemotePlaceCache.cache.get(CACHE_KEY); - if (!placeCache) { - return; - } - if (!domain) { - placeCache = {}; - } else if (!contactTypeName) { - delete placeCache[domain]; - } else { - delete placeCache[domain][contactTypeName]; - } - RemotePlaceCache.cache.set(CACHE_KEY, placeCache); - } - - private static async getDomainStore(chtApi: ChtApi, placeType: string) - : Promise { - const { domain } = chtApi.chtSession.authInfo; - let domainCache = RemotePlaceCache.cache.get(CACHE_KEY); - if (!domainCache) { - domainCache = {}; - } - - const places = domainCache[domain]?.[placeType]; - if (!places) { - const fetchPlacesWithType = chtApi.getPlacesWithType(placeType); - domainCache[domain] = { - ...domainCache[domain], - [placeType]: await fetchPlacesWithType, - }; - RemotePlaceCache.cache.set(CACHE_KEY, domainCache); - } - - return domainCache[domain][placeType]; - } -} diff --git a/src/lib/remote-place-resolver.ts b/src/lib/remote-place-resolver.ts index e7d10a57..209e1735 100644 --- a/src/lib/remote-place-resolver.ts +++ b/src/lib/remote-place-resolver.ts @@ -4,7 +4,6 @@ import SessionCache from '../services/session-cache'; import { RemotePlace, ChtApi } from './cht-api'; import { Config, ContactType, HierarchyConstraint } from '../config'; import { Validation } from './validation'; -import RemotePlaceCache from './remote-place-cache'; type RemotePlaceMap = { [key: string]: RemotePlace }; @@ -123,7 +122,7 @@ async function findRemotePlacesInHierarchy( hierarchyLevel: HierarchyConstraint, chtApi: ChtApi ) : Promise { - let searchPool = await RemotePlaceCache.getPlacesWithType(chtApi, hierarchyLevel.contact_type); + let searchPool = await chtApi.getPlacesWithType(hierarchyLevel.contact_type); searchPool = searchPool.filter(remotePlace => chtApi.chtSession.isPlaceAuthorized(remotePlace)); const topDownHierarchy = Config.getHierarchyWithReplacement(place.type, 'desc'); diff --git a/src/lib/search.ts b/src/lib/search.ts index dee99799..88472415 100644 --- a/src/lib/search.ts +++ b/src/lib/search.ts @@ -1,7 +1,6 @@ import _ from 'lodash'; import SessionCache from '../services/session-cache'; import { ChtApi, RemotePlace } from './cht-api'; -import RemotePlaceCache from './remote-place-cache'; import RemotePlaceResolver from './remote-place-resolver'; import { Config, ContactType, HierarchyConstraint } from '../config'; import Place from '../services/place'; @@ -56,7 +55,7 @@ async function getRemoteResults( chtApi: ChtApi, dataPrefix: string ) : Promise { - let remoteResults = (await RemotePlaceCache.getPlacesWithType(chtApi, hierarchyLevel.contact_type)) + let remoteResults = (await chtApi.getPlacesWithType(hierarchyLevel.contact_type)) .filter(remotePlace => chtApi.chtSession.isPlaceAuthorized(remotePlace)) .filter(place => place.name.includes(searchString)); @@ -71,7 +70,7 @@ async function getRemoteResults( continue; } - const placesAtLevel = await RemotePlaceCache.getPlacesWithType(chtApi, constrainingHierarchy.contact_type); + const placesAtLevel = await chtApi.getPlacesWithType(constrainingHierarchy.contact_type); const relevantPlaceIds = placesAtLevel .filter(remotePlace => remotePlace.name.includes(searchStringAtLevel)) .map(remotePlace => remotePlace.id); diff --git a/src/liquid/app/nav.html b/src/liquid/app/nav.html index 533ef2d8..ac5ef0db 100644 --- a/src/liquid/app/nav.html +++ b/src/liquid/app/nav.html @@ -47,9 +47,6 @@