perf measurement & query facility (#104)

.changeset/orange-meals-poke.md

@@ -0,0 +1,5 @@
+---
+'contentsgarten': minor
+---
+
+The `view` method now returns a `perf` array which contains the performance logs

.changeset/stale-fans-shake.md

@@ -0,0 +1,5 @@
+---
+'contentsgarten': minor
+---
+
+Add ability to query page data

packages/contentsgarten/src/Contentsgarten.ts

@@ -6,6 +6,7 @@ import { QueryClient } from '@tanstack/query-core'
 import type { ContentsgartenConfig } from './ContentsgartenConfig'
 import { fetchRequestHandler } from '@trpc/server/adapters/fetch'
 import { ContentsgartenRouter } from './ContentsgartenRouter'
+import { PerfContextImpl } from './PerfContextImpl'
 
 export class Contentsgarten {
   private globalContext: ContentsgartenAppContext
@@ -27,6 +28,7 @@ export class Contentsgarten {
         defaultOptions: { queries: { staleTime: Infinity } },
       }),
       authToken: input.authToken,
+      perf: new PerfContextImpl(),
     }
   }
 }

packages/contentsgarten/src/ContentsgartenPageDatabase.ts

@@ -1,21 +1,83 @@
 import { Document, Db } from 'mongodb'
+import { z } from 'zod'
+import { PageRefRegex } from './PageRefRegex'
+import { escapeRegExp } from 'lodash-es'
 
 const PageCollection = defineCollectionSchema<PageDoc>('pages')
 const currentCacheVersion = 'v3'
 
+export const PageDatabaseQuery = z.object({
+  match: z
+    .record(z.union([z.string(), z.array(z.string())]))
+    .optional()
+    .describe(
+      'Only return pages that have front-matter matching these properties.',
+    ),
+  prefix: z
+    .string()
+    .regex(PageRefRegex)
+    .regex(/\/$/, 'Prefix must end with a slash')
+    .optional()
+    .describe(
+      'Only return pages with this prefix. The prefix must end with a slash.',
+    ),
+})
+export type PageDatabaseQuery = z.infer<typeof PageDatabaseQuery>
+export interface PageDatabaseQueryResult {
+  count: number
+  results: PageDatabaseQueryResultItem[]
+  explain?: any
+}
+export interface PageDatabaseQueryResultItem {
+  lastModified: Date | null
+  pageRef: string
+  frontMatter: Record<string, any>
+}
+
 export interface ContentsgartenPageDatabase {
   getCached(pageRef: string): Promise<PageData | null>
   save(pageRef: string, input: PageDataInput): Promise<PageData>
   getRecentlyChangedPages(): Promise<PageListingItem[]>
+  queryPages(query: PageDatabaseQuery): Promise<PageDatabaseQueryResult>
+  checkTypo(normalized: string): Promise<string[]>
 }
 
 export interface PageListingItem {
   pageRef: string
   lastModified: Date
 }
+const explainResult = z.object({
+  executionStats: z.object({
+    executionTimeMillis: z.any(),
+    totalKeysExamined: z.any(),
+    totalDocsExamined: z.any(),
+  }),
+})
 
 export class MongoDBPageDatabase implements ContentsgartenPageDatabase {
-  constructor(private db: Db) {}
+  constructor(private db: Db) {
+    this.createIndex().catch((err) => {
+      console.error('Failed to create index', err)
+    })
+  }
+  async createIndex() {
+    const collection = PageCollection.of(this.db)
+    return Promise.all([
+      collection.createIndex({
+        cacheVersion: 1,
+        lastModified: -1,
+      }),
+      collection.createIndex({
+        cacheVersion: 1,
+        'aux.keyValues': 1,
+        lastModified: -1,
+      }),
+      collection.createIndex({
+        cacheVersion: 1,
+        'aux.normalized': 1,
+      }),
+    ])
+  }
   async getCached(pageRef: string): Promise<PageData | null> {
     const collection = PageCollection.of(this.db)
     const doc = await collection.findOne({
@@ -46,6 +108,7 @@ export class MongoDBPageDatabase implements ContentsgartenPageDatabase {
     const docs = await collection
       .find({
         cacheVersion: currentCacheVersion,
+        data: { $ne: null },
         lastModified: { $ne: null },
       })
       .sort({ lastModified: -1 })
@@ -56,6 +119,72 @@ export class MongoDBPageDatabase implements ContentsgartenPageDatabase {
       lastModified: doc.lastModified!,
     }))
   }
+  async queryPages(input: PageDatabaseQuery): Promise<PageDatabaseQueryResult> {
+    const collection = PageCollection.of(this.db)
+    const filter = compileQuery(input)
+    const cursor = collection
+      .find(filter)
+      .project({
+        _id: 1,
+        lastModified: 1,
+        'aux.frontmatter': 1,
+      })
+      .limit(1000)
+    const results = (await cursor.toArray()) as PageDoc[]
+    return {
+      count: results.length,
+      results: results.map((doc) => ({
+        pageRef: doc._id,
+        lastModified: doc.lastModified,
+        frontMatter: doc.aux.frontmatter,
+      })),
+      explain: {
+        filter: filter,
+        executionStats: explainResult.parse(
+          await cursor.explain('executionStats'),
+        ),
+      },
+    }
+  }
+  async checkTypo(normalized: string): Promise<string[]> {
+    const collection = PageCollection.of(this.db)
+    const cursor = collection
+      .find({
+        cacheVersion: currentCacheVersion,
+        'aux.normalized': normalized,
+        data: { $ne: null },
+      })
+      .project({ _id: 1 })
+      .limit(10)
+    const results = await cursor.toArray()
+    return results.map((doc) => doc._id)
+  }
+}
+
+function compileQuery(input: PageDatabaseQuery): any {
+  let ands: any[] = [
+    {
+      cacheVersion: currentCacheVersion,
+      data: { $ne: null },
+    },
+  ]
+  if (input.match) {
+    ands.push(
+      ...Object.entries(input.match).map(([key, value]) => {
+        const f = (v: string) => `${key}=${v}`
+        if (Array.isArray(value)) {
+          return { 'aux.keyValues': { $in: value.map(f) } }
+        }
+        return { 'aux.keyValues': f(value) }
+      }),
+    )
+  }
+  if (input.prefix) {
+    ands.push({
+      _id: { $regex: `^${escapeRegExp(input.prefix)}` },
+    })
+  }
+  return { $and: ands }
 }
 
 export interface PageDoc {
@@ -75,6 +204,8 @@ export interface PageDocFile {
 }
 export interface PageAuxiliaryData {
   frontmatter: Record<string, any>
+  normalized?: string
+  keyValues?: string[]
 }
 
 export type PageData = Pick<

packages/contentsgarten/src/ContentsgartenRouter.ts

@@ -14,11 +14,13 @@ import {
 } from './getPage'
 import { load } from 'js-yaml'
 import { cache } from './cache'
-import { PageRefRegex } from './PageRefRegex'
+import { LaxPageRefRegex, PageRefRegex } from './PageRefRegex'
+import { PageDatabaseQuery } from './ContentsgartenPageDatabase'
 
 export { GetPageResult } from './getPage'
 export { PageRefRegex }
 export const PageRef = z.string().regex(PageRefRegex)
+export const LaxPageRef = z.string().regex(LaxPageRefRegex)
 
 export const ContentsgartenRouter = t.router({
   about: t.procedure
@@ -56,18 +58,18 @@ export const ContentsgartenRouter = t.router({
     .meta({ summary: 'Returns the page information' })
     .input(
       z.object({
-        pageRef: PageRef,
+        pageRef: LaxPageRef,
         withFile: z.boolean().default(true),
         revalidate: z.boolean().optional(),
         render: z.boolean().optional(),
       }),
     )
-    .output(GetPageResult)
+    .output(GetPageResult.merge(z.object({ perf: z.array(z.string()) })))
     .query(
       async ({ input: { pageRef, revalidate, withFile, render }, ctx }) => {
         const page = await getPage(ctx, pageRef, revalidate, render)
         const result: GetPageResult = withFile ? page : omit(page, 'file')
-        return result
+        return { ...result, perf: ctx.perf.toMessageArray() }
       },
     ),
   getEditPermission: t.procedure
@@ -147,6 +149,15 @@ export const ContentsgartenRouter = t.router({
       })
       return { revision: result.revision }
     }),
+  query: t.procedure
+    .meta({
+      summary:
+        'Runs a query against the pages in database. Most recently updated pages are returned first.',
+    })
+    .input(PageDatabaseQuery)
+    .query(async ({ input, ctx }) => {
+      return await ctx.app.pageDatabase.queryPages(input)
+    }),
 })
 
 export type ContentsgartenRouter = typeof ContentsgartenRouter

packages/contentsgarten/src/PageRefRegex.ts

@@ -1 +1,2 @@
 export const PageRefRegex = /^[A-Z][A-Za-z0-9_/-]*$/
+export const LaxPageRefRegex = /^[A-Za-z0-9_/-]+$/

packages/contentsgarten/src/PerfContextImpl.ts

@@ -0,0 +1,48 @@
+import { PerfContext, PerfEntry } from './RequestContext'
+
+export class PerfContextImpl implements PerfContext {
+  private begin = performance.now()
+  private beginTime = new Date()
+  private data: { start: number; finish?: number; text: string }[] = []
+  async measure<T>(
+    name: string,
+    fn: (entry: PerfEntry) => PromiseLike<T>,
+  ): Promise<T> {
+    const entry: (typeof this.data)[number] = {
+      start: performance.now(),
+      text: name,
+    }
+    this.data.push(entry)
+    try {
+      return await fn({
+        addInfo: (info) => {
+          entry.text += ` (${info})`
+        },
+      })
+    } finally {
+      entry.finish = performance.now()
+    }
+  }
+  log(name: string): void {
+    this.data.push({ start: performance.now(), text: name })
+  }
+  toMessageArray() {
+    const out: string[] = []
+    out.push(`start: ${this.beginTime.toISOString()}`)
+    const begin = this.begin
+    const fmt = (d: number) => `${(d / 1000).toFixed(3)}s`
+    const t = (ts: number) => fmt(ts - begin)
+    for (const { start, finish, text } of this.data) {
+      if (finish) {
+        out.push(
+          `${t(start)} ~ ${t(finish)} | ${text} (took ${fmt(finish - start)})`,
+        )
+      } else {
+        out.push(`${t(start)} | ${text}`)
+      }
+    }
+    const finish = performance.now()
+    out.push(`${t(finish)} | finish (total ${fmt(finish - begin)}))`)
+    return out
+  }
+}

packages/contentsgarten/src/RequestContext.ts

@@ -3,8 +3,19 @@ import type { QueryClient } from '@tanstack/query-core'
 export interface RequestContext {
   queryClient: QueryClient
   app: AppContext
+  perf: PerfContext
 }
 
 export interface AppContext {
   queryClient: QueryClient
 }
+
+export interface PerfContext {
+  measure<T>(name: string, fn: (entry: PerfEntry) => PromiseLike<T>): Promise<T>
+  log(name: string): void
+  toMessageArray(): string[]
+}
+
+export interface PerfEntry {
+  addInfo(info: string): void
+}

packages/contentsgarten/src/cache.ts

@@ -6,14 +6,17 @@ export async function cache<T>(
   f: () => Promise<T>,
   ttl: number,
 ): Promise<T> {
-  return ctx.app.queryClient.fetchQuery({
-    queryKey: ['cache', cacheKey],
-    queryFn: async () => {
-      const result = await f()
-      return result
-    },
-    staleTime: ttl,
-  })
+  return ctx.perf.measure(`cache(${cacheKey}, ttl=${ttl})`, (e) =>
+    ctx.app.queryClient.fetchQuery({
+      queryKey: ['cache', cacheKey],
+      queryFn: async () => {
+        e.addInfo('MISS')
+        const result = await f()
+        return result
+      },
+      staleTime: ttl,
+    }),
+  )
 }
 
 export async function staleOrRevalidate<T>(