Skip to content

Commit aebab74

Browse files
dtinthdtinth
authored
perf measurement & query facility (#104)
1 parent 1010b18 commit aebab74

File tree

11 files changed

+344
-53
lines changed

11 files changed

+344
-53
lines changed

‎.changeset/orange-meals-poke.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'contentsgarten': minor
3+
---
4+
5+
The `view` method now returns a `perf` array which contains the performance logs

‎.changeset/stale-fans-shake.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'contentsgarten': minor
3+
---
4+
5+
Add ability to query page data

‎packages/contentsgarten/src/Contentsgarten.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { QueryClient } from '@tanstack/query-core'
66
import type { ContentsgartenConfig } from './ContentsgartenConfig'
77
import { fetchRequestHandler } from '@trpc/server/adapters/fetch'
88
import { ContentsgartenRouter } from './ContentsgartenRouter'
9+
import { PerfContextImpl } from './PerfContextImpl'
910

1011
export class Contentsgarten {
1112
private globalContext: ContentsgartenAppContext
@@ -27,6 +28,7 @@ export class Contentsgarten {
2728
defaultOptions: { queries: { staleTime: Infinity } },
2829
}),
2930
authToken: input.authToken,
31+
perf: new PerfContextImpl(),
3032
}
3133
}
3234
}

‎packages/contentsgarten/src/ContentsgartenPageDatabase.ts‎

Lines changed: 132 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,83 @@
11
import { Document, Db } from 'mongodb'
2+
import { z } from 'zod'
3+
import { PageRefRegex } from './PageRefRegex'
4+
import { escapeRegExp } from 'lodash-es'
25

36
const PageCollection = defineCollectionSchema<PageDoc>('pages')
47
const currentCacheVersion = 'v3'
58

9+
export const PageDatabaseQuery = z.object({
10+
match: z
11+
.record(z.union([z.string(), z.array(z.string())]))
12+
.optional()
13+
.describe(
14+
'Only return pages that have front-matter matching these properties.',
15+
),
16+
prefix: z
17+
.string()
18+
.regex(PageRefRegex)
19+
.regex(/\/$/, 'Prefix must end with a slash')
20+
.optional()
21+
.describe(
22+
'Only return pages with this prefix. The prefix must end with a slash.',
23+
),
24+
})
25+
export type PageDatabaseQuery = z.infer<typeof PageDatabaseQuery>
26+
export interface PageDatabaseQueryResult {
27+
count: number
28+
results: PageDatabaseQueryResultItem[]
29+
explain?: any
30+
}
31+
export interface PageDatabaseQueryResultItem {
32+
lastModified: Date | null
33+
pageRef: string
34+
frontMatter: Record<string, any>
35+
}
36+
637
export interface ContentsgartenPageDatabase {
738
getCached(pageRef: string): Promise<PageData | null>
839
save(pageRef: string, input: PageDataInput): Promise<PageData>
940
getRecentlyChangedPages(): Promise<PageListingItem[]>
41+
queryPages(query: PageDatabaseQuery): Promise<PageDatabaseQueryResult>
42+
checkTypo(normalized: string): Promise<string[]>
1043
}
1144

1245
export interface PageListingItem {
1346
pageRef: string
1447
lastModified: Date
1548
}
49+
const explainResult = z.object({
50+
executionStats: z.object({
51+
executionTimeMillis: z.any(),
52+
totalKeysExamined: z.any(),
53+
totalDocsExamined: z.any(),
54+
}),
55+
})
1656

1757
export class MongoDBPageDatabase implements ContentsgartenPageDatabase {
18-
constructor(private db: Db) {}
58+
constructor(private db: Db) {
59+
this.createIndex().catch((err) => {
60+
console.error('Failed to create index', err)
61+
})
62+
}
63+
async createIndex() {
64+
const collection = PageCollection.of(this.db)
65+
return Promise.all([
66+
collection.createIndex({
67+
cacheVersion: 1,
68+
lastModified: -1,
69+
}),
70+
collection.createIndex({
71+
cacheVersion: 1,
72+
'aux.keyValues': 1,
73+
lastModified: -1,
74+
}),
75+
collection.createIndex({
76+
cacheVersion: 1,
77+
'aux.normalized': 1,
78+
}),
79+
])
80+
}
1981
async getCached(pageRef: string): Promise<PageData | null> {
2082
const collection = PageCollection.of(this.db)
2183
const doc = await collection.findOne({
@@ -46,6 +108,7 @@ export class MongoDBPageDatabase implements ContentsgartenPageDatabase {
46108
const docs = await collection
47109
.find({
48110
cacheVersion: currentCacheVersion,
111+
data: { $ne: null },
49112
lastModified: { $ne: null },
50113
})
51114
.sort({ lastModified: -1 })
@@ -56,6 +119,72 @@ export class MongoDBPageDatabase implements ContentsgartenPageDatabase {
56119
lastModified: doc.lastModified!,
57120
}))
58121
}
122+
async queryPages(input: PageDatabaseQuery): Promise<PageDatabaseQueryResult> {
123+
const collection = PageCollection.of(this.db)
124+
const filter = compileQuery(input)
125+
const cursor = collection
126+
.find(filter)
127+
.project({
128+
_id: 1,
129+
lastModified: 1,
130+
'aux.frontmatter': 1,
131+
})
132+
.limit(1000)
133+
const results = (await cursor.toArray()) as PageDoc[]
134+
return {
135+
count: results.length,
136+
results: results.map((doc) => ({
137+
pageRef: doc._id,
138+
lastModified: doc.lastModified,
139+
frontMatter: doc.aux.frontmatter,
140+
})),
141+
explain: {
142+
filter: filter,
143+
executionStats: explainResult.parse(
144+
await cursor.explain('executionStats'),
145+
),
146+
},
147+
}
148+
}
149+
async checkTypo(normalized: string): Promise<string[]> {
150+
const collection = PageCollection.of(this.db)
151+
const cursor = collection
152+
.find({
153+
cacheVersion: currentCacheVersion,
154+
'aux.normalized': normalized,
155+
data: { $ne: null },
156+
})
157+
.project({ _id: 1 })
158+
.limit(10)
159+
const results = await cursor.toArray()
160+
return results.map((doc) => doc._id)
161+
}
162+
}
163+
164+
function compileQuery(input: PageDatabaseQuery): any {
165+
let ands: any[] = [
166+
{
167+
cacheVersion: currentCacheVersion,
168+
data: { $ne: null },
169+
},
170+
]
171+
if (input.match) {
172+
ands.push(
173+
...Object.entries(input.match).map(([key, value]) => {
174+
const f = (v: string) => `${key}=${v}`
175+
if (Array.isArray(value)) {
176+
return { 'aux.keyValues': { $in: value.map(f) } }
177+
}
178+
return { 'aux.keyValues': f(value) }
179+
}),
180+
)
181+
}
182+
if (input.prefix) {
183+
ands.push({
184+
_id: { $regex: `^${escapeRegExp(input.prefix)}` },
185+
})
186+
}
187+
return { $and: ands }
59188
}
60189

61190
export interface PageDoc {
@@ -75,6 +204,8 @@ export interface PageDocFile {
75204
}
76205
export interface PageAuxiliaryData {
77206
frontmatter: Record<string, any>
207+
normalized?: string
208+
keyValues?: string[]
78209
}
79210

80211
export type PageData = Pick<

‎packages/contentsgarten/src/ContentsgartenRouter.ts‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ import {
1414
} from './getPage'
1515
import { load } from 'js-yaml'
1616
import { cache } from './cache'
17-
import { PageRefRegex } from './PageRefRegex'
17+
import { LaxPageRefRegex, PageRefRegex } from './PageRefRegex'
18+
import { PageDatabaseQuery } from './ContentsgartenPageDatabase'
1819

1920
export { GetPageResult } from './getPage'
2021
export { PageRefRegex }
2122
export const PageRef = z.string().regex(PageRefRegex)
23+
export const LaxPageRef = z.string().regex(LaxPageRefRegex)
2224

2325
export const ContentsgartenRouter = t.router({
2426
about: t.procedure
@@ -56,18 +58,18 @@ export const ContentsgartenRouter = t.router({
5658
.meta({ summary: 'Returns the page information' })
5759
.input(
5860
z.object({
59-
pageRef: PageRef,
61+
pageRef: LaxPageRef,
6062
withFile: z.boolean().default(true),
6163
revalidate: z.boolean().optional(),
6264
render: z.boolean().optional(),
6365
}),
6466
)
65-
.output(GetPageResult)
67+
.output(GetPageResult.merge(z.object({ perf: z.array(z.string()) })))
6668
.query(
6769
async ({ input: { pageRef, revalidate, withFile, render }, ctx }) => {
6870
const page = await getPage(ctx, pageRef, revalidate, render)
6971
const result: GetPageResult = withFile ? page : omit(page, 'file')
70-
return result
72+
return { ...result, perf: ctx.perf.toMessageArray() }
7173
},
7274
),
7375
getEditPermission: t.procedure
@@ -147,6 +149,15 @@ export const ContentsgartenRouter = t.router({
147149
})
148150
return { revision: result.revision }
149151
}),
152+
query: t.procedure
153+
.meta({
154+
summary:
155+
'Runs a query against the pages in database. Most recently updated pages are returned first.',
156+
})
157+
.input(PageDatabaseQuery)
158+
.query(async ({ input, ctx }) => {
159+
return await ctx.app.pageDatabase.queryPages(input)
160+
}),
150161
})
151162

152163
export type ContentsgartenRouter = typeof ContentsgartenRouter

‎packages/contentsgarten/src/PageRefRegex.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
export const PageRefRegex = /^[A-Z][A-Za-z0-9_/-]*$/
2+
export const LaxPageRefRegex = /^[A-Za-z0-9_/-]+$/

‎packages/contentsgarten/src/PerfContextImpl.ts‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import { PerfContext, PerfEntry } from './RequestContext'
2+
3+
export class PerfContextImpl implements PerfContext {
4+
private begin = performance.now()
5+
private beginTime = new Date()
6+
private data: { start: number; finish?: number; text: string }[] = []
7+
async measure<T>(
8+
name: string,
9+
fn: (entry: PerfEntry) => PromiseLike<T>,
10+
): Promise<T> {
11+
const entry: (typeof this.data)[number] = {
12+
start: performance.now(),
13+
text: name,
14+
}
15+
this.data.push(entry)
16+
try {
17+
return await fn({
18+
addInfo: (info) => {
19+
entry.text += ` (${info})`
20+
},
21+
})
22+
} finally {
23+
entry.finish = performance.now()
24+
}
25+
}
26+
log(name: string): void {
27+
this.data.push({ start: performance.now(), text: name })
28+
}
29+
toMessageArray() {
30+
const out: string[] = []
31+
out.push(`start: ${this.beginTime.toISOString()}`)
32+
const begin = this.begin
33+
const fmt = (d: number) => `${(d / 1000).toFixed(3)}s`
34+
const t = (ts: number) => fmt(ts - begin)
35+
for (const { start, finish, text } of this.data) {
36+
if (finish) {
37+
out.push(
38+
`${t(start)} ~ ${t(finish)} | ${text} (took ${fmt(finish - start)})`,
39+
)
40+
} else {
41+
out.push(`${t(start)} | ${text}`)
42+
}
43+
}
44+
const finish = performance.now()
45+
out.push(`${t(finish)} | finish (total ${fmt(finish - begin)}))`)
46+
return out
47+
}
48+
}

‎packages/contentsgarten/src/RequestContext.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,19 @@ import type { QueryClient } from '@tanstack/query-core'
33
export interface RequestContext {
44
queryClient: QueryClient
55
app: AppContext
6+
perf: PerfContext
67
}
78

89
export interface AppContext {
910
queryClient: QueryClient
1011
}
12+
13+
export interface PerfContext {
14+
measure<T>(name: string, fn: (entry: PerfEntry) => PromiseLike<T>): Promise<T>
15+
log(name: string): void
16+
toMessageArray(): string[]
17+
}
18+
19+
export interface PerfEntry {
20+
addInfo(info: string): void
21+
}

‎packages/contentsgarten/src/cache.ts‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,17 @@ export async function cache<T>(
66
f: () => Promise<T>,
77
ttl: number,
88
): Promise<T> {
9-
return ctx.app.queryClient.fetchQuery({
10-
queryKey: ['cache', cacheKey],
11-
queryFn: async () => {
12-
const result = await f()
13-
return result
14-
},
15-
staleTime: ttl,
16-
})
9+
return ctx.perf.measure(`cache(${cacheKey}, ttl=${ttl})`, (e) =>
10+
ctx.app.queryClient.fetchQuery({
11+
queryKey: ['cache', cacheKey],
12+
queryFn: async () => {
13+
e.addInfo('MISS')
14+
const result = await f()
15+
return result
16+
},
17+
staleTime: ttl,
18+
}),
19+
)
1720
}
1821

1922
export async function staleOrRevalidate<T>(

0 commit comments

Comments
 (0)