diff --git a/.changeset/orange-meals-poke.md b/.changeset/orange-meals-poke.md new file mode 100644 index 00000000..b405a7b6 --- /dev/null +++ b/.changeset/orange-meals-poke.md @@ -0,0 +1,5 @@ +--- +'contentsgarten': minor +--- + +The `view` method now returns a `perf` array which contains the performance logs \ No newline at end of file diff --git a/.changeset/stale-fans-shake.md b/.changeset/stale-fans-shake.md new file mode 100644 index 00000000..4bbb085d --- /dev/null +++ b/.changeset/stale-fans-shake.md @@ -0,0 +1,5 @@ +--- +'contentsgarten': minor +--- + +Add ability to query page data diff --git a/packages/contentsgarten/src/Contentsgarten.ts b/packages/contentsgarten/src/Contentsgarten.ts index 12e41df9..6b6bd4f4 100644 --- a/packages/contentsgarten/src/Contentsgarten.ts +++ b/packages/contentsgarten/src/Contentsgarten.ts @@ -6,6 +6,7 @@ import { QueryClient } from '@tanstack/query-core' import type { ContentsgartenConfig } from './ContentsgartenConfig' import { fetchRequestHandler } from '@trpc/server/adapters/fetch' import { ContentsgartenRouter } from './ContentsgartenRouter' +import { PerfContextImpl } from './PerfContextImpl' export class Contentsgarten { private globalContext: ContentsgartenAppContext @@ -27,6 +28,7 @@ export class Contentsgarten { defaultOptions: { queries: { staleTime: Infinity } }, }), authToken: input.authToken, + perf: new PerfContextImpl(), } } } diff --git a/packages/contentsgarten/src/ContentsgartenPageDatabase.ts b/packages/contentsgarten/src/ContentsgartenPageDatabase.ts index 4aba61d5..63e248ff 100644 --- a/packages/contentsgarten/src/ContentsgartenPageDatabase.ts +++ b/packages/contentsgarten/src/ContentsgartenPageDatabase.ts @@ -1,21 +1,83 @@ import { Document, Db } from 'mongodb' +import { z } from 'zod' +import { PageRefRegex } from './PageRefRegex' +import { escapeRegExp } from 'lodash-es' const PageCollection = defineCollectionSchema('pages') const currentCacheVersion = 'v3' +export const PageDatabaseQuery = z.object({ + match: z + .record(z.union([z.string(), z.array(z.string())])) + .optional() + .describe( + 'Only return pages that have front-matter matching these properties.', + ), + prefix: z + .string() + .regex(PageRefRegex) + .regex(/\/$/, 'Prefix must end with a slash') + .optional() + .describe( + 'Only return pages with this prefix. The prefix must end with a slash.', + ), +}) +export type PageDatabaseQuery = z.infer +export interface PageDatabaseQueryResult { + count: number + results: PageDatabaseQueryResultItem[] + explain?: any +} +export interface PageDatabaseQueryResultItem { + lastModified: Date | null + pageRef: string + frontMatter: Record +} + export interface ContentsgartenPageDatabase { getCached(pageRef: string): Promise save(pageRef: string, input: PageDataInput): Promise getRecentlyChangedPages(): Promise + queryPages(query: PageDatabaseQuery): Promise + checkTypo(normalized: string): Promise } export interface PageListingItem { pageRef: string lastModified: Date } +const explainResult = z.object({ + executionStats: z.object({ + executionTimeMillis: z.any(), + totalKeysExamined: z.any(), + totalDocsExamined: z.any(), + }), +}) export class MongoDBPageDatabase implements ContentsgartenPageDatabase { - constructor(private db: Db) {} + constructor(private db: Db) { + this.createIndex().catch((err) => { + console.error('Failed to create index', err) + }) + } + async createIndex() { + const collection = PageCollection.of(this.db) + return Promise.all([ + collection.createIndex({ + cacheVersion: 1, + lastModified: -1, + }), + collection.createIndex({ + cacheVersion: 1, + 'aux.keyValues': 1, + lastModified: -1, + }), + collection.createIndex({ + cacheVersion: 1, + 'aux.normalized': 1, + }), + ]) + } async getCached(pageRef: string): Promise { const collection = PageCollection.of(this.db) const doc = await collection.findOne({ @@ -46,6 +108,7 @@ export class MongoDBPageDatabase implements ContentsgartenPageDatabase { const docs = await collection .find({ cacheVersion: currentCacheVersion, + data: { $ne: null }, lastModified: { $ne: null }, }) .sort({ lastModified: -1 }) @@ -56,6 +119,72 @@ export class MongoDBPageDatabase implements ContentsgartenPageDatabase { lastModified: doc.lastModified!, })) } + async queryPages(input: PageDatabaseQuery): Promise { + const collection = PageCollection.of(this.db) + const filter = compileQuery(input) + const cursor = collection + .find(filter) + .project({ + _id: 1, + lastModified: 1, + 'aux.frontmatter': 1, + }) + .limit(1000) + const results = (await cursor.toArray()) as PageDoc[] + return { + count: results.length, + results: results.map((doc) => ({ + pageRef: doc._id, + lastModified: doc.lastModified, + frontMatter: doc.aux.frontmatter, + })), + explain: { + filter: filter, + executionStats: explainResult.parse( + await cursor.explain('executionStats'), + ), + }, + } + } + async checkTypo(normalized: string): Promise { + const collection = PageCollection.of(this.db) + const cursor = collection + .find({ + cacheVersion: currentCacheVersion, + 'aux.normalized': normalized, + data: { $ne: null }, + }) + .project({ _id: 1 }) + .limit(10) + const results = await cursor.toArray() + return results.map((doc) => doc._id) + } +} + +function compileQuery(input: PageDatabaseQuery): any { + let ands: any[] = [ + { + cacheVersion: currentCacheVersion, + data: { $ne: null }, + }, + ] + if (input.match) { + ands.push( + ...Object.entries(input.match).map(([key, value]) => { + const f = (v: string) => `${key}=${v}` + if (Array.isArray(value)) { + return { 'aux.keyValues': { $in: value.map(f) } } + } + return { 'aux.keyValues': f(value) } + }), + ) + } + if (input.prefix) { + ands.push({ + _id: { $regex: `^${escapeRegExp(input.prefix)}` }, + }) + } + return { $and: ands } } export interface PageDoc { @@ -75,6 +204,8 @@ export interface PageDocFile { } export interface PageAuxiliaryData { frontmatter: Record + normalized?: string + keyValues?: string[] } export type PageData = Pick< diff --git a/packages/contentsgarten/src/ContentsgartenRouter.ts b/packages/contentsgarten/src/ContentsgartenRouter.ts index 5faa922a..b38a2735 100644 --- a/packages/contentsgarten/src/ContentsgartenRouter.ts +++ b/packages/contentsgarten/src/ContentsgartenRouter.ts @@ -14,11 +14,13 @@ import { } from './getPage' import { load } from 'js-yaml' import { cache } from './cache' -import { PageRefRegex } from './PageRefRegex' +import { LaxPageRefRegex, PageRefRegex } from './PageRefRegex' +import { PageDatabaseQuery } from './ContentsgartenPageDatabase' export { GetPageResult } from './getPage' export { PageRefRegex } export const PageRef = z.string().regex(PageRefRegex) +export const LaxPageRef = z.string().regex(LaxPageRefRegex) export const ContentsgartenRouter = t.router({ about: t.procedure @@ -56,18 +58,18 @@ export const ContentsgartenRouter = t.router({ .meta({ summary: 'Returns the page information' }) .input( z.object({ - pageRef: PageRef, + pageRef: LaxPageRef, withFile: z.boolean().default(true), revalidate: z.boolean().optional(), render: z.boolean().optional(), }), ) - .output(GetPageResult) + .output(GetPageResult.merge(z.object({ perf: z.array(z.string()) }))) .query( async ({ input: { pageRef, revalidate, withFile, render }, ctx }) => { const page = await getPage(ctx, pageRef, revalidate, render) const result: GetPageResult = withFile ? page : omit(page, 'file') - return result + return { ...result, perf: ctx.perf.toMessageArray() } }, ), getEditPermission: t.procedure @@ -147,6 +149,15 @@ export const ContentsgartenRouter = t.router({ }) return { revision: result.revision } }), + query: t.procedure + .meta({ + summary: + 'Runs a query against the pages in database. Most recently updated pages are returned first.', + }) + .input(PageDatabaseQuery) + .query(async ({ input, ctx }) => { + return await ctx.app.pageDatabase.queryPages(input) + }), }) export type ContentsgartenRouter = typeof ContentsgartenRouter diff --git a/packages/contentsgarten/src/PageRefRegex.ts b/packages/contentsgarten/src/PageRefRegex.ts index f1f082a1..fe2828a1 100644 --- a/packages/contentsgarten/src/PageRefRegex.ts +++ b/packages/contentsgarten/src/PageRefRegex.ts @@ -1 +1,2 @@ export const PageRefRegex = /^[A-Z][A-Za-z0-9_/-]*$/ +export const LaxPageRefRegex = /^[A-Za-z0-9_/-]+$/ diff --git a/packages/contentsgarten/src/PerfContextImpl.ts b/packages/contentsgarten/src/PerfContextImpl.ts new file mode 100644 index 00000000..17dea518 --- /dev/null +++ b/packages/contentsgarten/src/PerfContextImpl.ts @@ -0,0 +1,48 @@ +import { PerfContext, PerfEntry } from './RequestContext' + +export class PerfContextImpl implements PerfContext { + private begin = performance.now() + private beginTime = new Date() + private data: { start: number; finish?: number; text: string }[] = [] + async measure( + name: string, + fn: (entry: PerfEntry) => PromiseLike, + ): Promise { + const entry: (typeof this.data)[number] = { + start: performance.now(), + text: name, + } + this.data.push(entry) + try { + return await fn({ + addInfo: (info) => { + entry.text += ` (${info})` + }, + }) + } finally { + entry.finish = performance.now() + } + } + log(name: string): void { + this.data.push({ start: performance.now(), text: name }) + } + toMessageArray() { + const out: string[] = [] + out.push(`start: ${this.beginTime.toISOString()}`) + const begin = this.begin + const fmt = (d: number) => `${(d / 1000).toFixed(3)}s` + const t = (ts: number) => fmt(ts - begin) + for (const { start, finish, text } of this.data) { + if (finish) { + out.push( + `${t(start)} ~ ${t(finish)} | ${text} (took ${fmt(finish - start)})`, + ) + } else { + out.push(`${t(start)} | ${text}`) + } + } + const finish = performance.now() + out.push(`${t(finish)} | finish (total ${fmt(finish - begin)}))`) + return out + } +} diff --git a/packages/contentsgarten/src/RequestContext.ts b/packages/contentsgarten/src/RequestContext.ts index 659aba39..f2a60a55 100644 --- a/packages/contentsgarten/src/RequestContext.ts +++ b/packages/contentsgarten/src/RequestContext.ts @@ -3,8 +3,19 @@ import type { QueryClient } from '@tanstack/query-core' export interface RequestContext { queryClient: QueryClient app: AppContext + perf: PerfContext } export interface AppContext { queryClient: QueryClient } + +export interface PerfContext { + measure(name: string, fn: (entry: PerfEntry) => PromiseLike): Promise + log(name: string): void + toMessageArray(): string[] +} + +export interface PerfEntry { + addInfo(info: string): void +} diff --git a/packages/contentsgarten/src/cache.ts b/packages/contentsgarten/src/cache.ts index 2016a8ab..efe8448f 100644 --- a/packages/contentsgarten/src/cache.ts +++ b/packages/contentsgarten/src/cache.ts @@ -6,14 +6,17 @@ export async function cache( f: () => Promise, ttl: number, ): Promise { - return ctx.app.queryClient.fetchQuery({ - queryKey: ['cache', cacheKey], - queryFn: async () => { - const result = await f() - return result - }, - staleTime: ttl, - }) + return ctx.perf.measure(`cache(${cacheKey}, ttl=${ttl})`, (e) => + ctx.app.queryClient.fetchQuery({ + queryKey: ['cache', cacheKey], + queryFn: async () => { + e.addInfo('MISS') + const result = await f() + return result + }, + staleTime: ttl, + }), + ) } export async function staleOrRevalidate( diff --git a/packages/contentsgarten/src/getPage.ts b/packages/contentsgarten/src/getPage.ts index 992c28ea..2e451934 100644 --- a/packages/contentsgarten/src/getPage.ts +++ b/packages/contentsgarten/src/getPage.ts @@ -2,7 +2,7 @@ import { ContentsgartenRequestContext } from './ContentsgartenContext' import { createLiquidEngine } from './createLiquidEngine' import { z } from 'zod' import { staleOrRevalidate } from './cache' -import { PageData } from './ContentsgartenPageDatabase' +import { PageData, PageAuxiliaryData } from './ContentsgartenPageDatabase' import matter from 'gray-matter' import { GetFileResult } from './ContentsgartenStorage' import { processMarkdown } from '@contentsgarten/markdown' @@ -48,7 +48,7 @@ export async function getPage( ) { if (pageRef.toLowerCase().startsWith('special/')) { return getSpecialPage(ctx, pageRef, revalidate).then((result) => - postProcess(result, render), + postProcess(ctx, result, render), ) } @@ -90,8 +90,16 @@ export async function getPage( const { content, frontMatter, status, targetPageRef } = await (async () => { if (!pageFile.data) { + const similarlyNamedPages = await ctx.perf.measure('checkTypo', () => + ctx.app.pageDatabase.checkTypo(normalizePageRef(pageRef)), + ) return { - content: '(This page currently does not exist.)', + content: + '(This page currently does not exist.)' + + (similarlyNamedPages.length > 0 + ? '\n\nDid you mean one of these pages?\n\n' + + similarlyNamedPages.map((page) => `- [[${page}]]`).join('\n') + : ''), frontMatter: {}, status: 404, } as const @@ -151,11 +159,19 @@ export async function getPage( lastModified: pageFile.lastModified?.toISOString() || undefined, lastModifiedBy: pageFile.lastModifiedBy, } - return postProcess(result, render) + return postProcess(ctx, result, render) } -function postProcess(result: GetPageResult, render: boolean): GetPageResult { - const rendered = render ? processMarkdown(result.content) : undefined +async function postProcess( + ctx: ContentsgartenRequestContext, + result: GetPageResult, + render: boolean, +): Promise { + const rendered = render + ? await ctx.perf.measure('processMarkdown', async () => + processMarkdown(result.content), + ) + : undefined return rendered ? { ...result, rendered } : result } @@ -177,23 +193,34 @@ export async function getPageFile( pageRef: string, revalidate = false, ) { - if (!revalidate) { - const cachedPage = await ctx.app.pageDatabase.getCached(pageRef) - if (cachedPage) { - return cachedPage - } - } - const page = await refreshPageFile(ctx, pageRef) - return page + return staleOrRevalidate( + ctx, + `page:${pageRef}`, + () => + ctx.perf.measure(`getPageFile(${pageRef})`, async (entry) => { + if (!revalidate) { + const cachedPage = await ctx.app.pageDatabase.getCached(pageRef) + if (cachedPage) { + return cachedPage + } + entry.addInfo('MISS') + } + const page = await refreshPageFile(ctx, pageRef) + return page + }), + revalidate ? 'revalidate' : 'stale', + ) } export async function refreshPageFile( ctx: ContentsgartenRequestContext, pageRef: string, ) { - const filePath = pageRefToFilePath(ctx, pageRef) - const getFileResult = (await ctx.app.storage.getFile(ctx, filePath)) || null - return savePageToDatabase(ctx, pageRef, getFileResult) + return ctx.perf.measure(`refreshPageFile(${pageRef})`, async (entry) => { + const filePath = pageRefToFilePath(ctx, pageRef) + const getFileResult = (await ctx.app.storage.getFile(ctx, filePath)) || null + return savePageToDatabase(ctx, pageRef, getFileResult) + }) } export async function savePageToDatabase( @@ -201,33 +228,71 @@ export async function savePageToDatabase( pageRef: string, getFileResult: GetFileResult | null, ) { - return ctx.app.pageDatabase.save( - pageRef, - getFileResult - ? { - data: { - contents: getFileResult.content.toString('utf8'), - revision: getFileResult.revision, - }, - lastModified: getFileResult.lastModified - ? new Date(getFileResult.lastModified) - : null, - lastModifiedBy: getFileResult.lastModifiedBy ?? [], - aux: { - frontmatter: matter(getFileResult.content.toString('utf8')).data, - }, - } - : { - data: null, - lastModified: null, - lastModifiedBy: [], - aux: { - frontmatter: {}, + return ctx.perf.measure(`savePageToDatabase(${pageRef})`, () => + ctx.app.pageDatabase.save( + pageRef, + getFileResult + ? { + data: { + contents: getFileResult.content.toString('utf8'), + revision: getFileResult.revision, + }, + lastModified: getFileResult.lastModified + ? new Date(getFileResult.lastModified) + : null, + lastModifiedBy: getFileResult.lastModifiedBy ?? [], + aux: getAux( + pageRef, + matter(getFileResult.content.toString('utf8')).data, + ), + } + : { + data: null, + lastModified: null, + lastModifiedBy: [], + aux: { frontmatter: {} }, }, - }, + ), ) } +function getAux(pageRef: string, frontMatter: any): PageAuxiliaryData { + return { + frontmatter: frontMatter, + keyValues: getKeyValues(frontMatter), + normalized: normalizePageRef(pageRef), + } +} + +function normalizePageRef(pageRef: string): string { + return pageRef.replace(/[_-]/g, '').toLowerCase() +} + +function getKeyValues(frontMatter: any): string[] { + const keyValues = new Set() + const traverse = (object: any, path: string[] = []): any => { + if ( + (typeof object === 'string' || + typeof object === 'number' || + typeof object === 'boolean' || + typeof object === 'bigint') && + path.length > 0 + ) { + keyValues.add(path.join('.') + '=' + object) + } else if (Array.isArray(object)) { + for (const item of object) { + traverse(item, path) + } + } else if (typeof object === 'object' && object) { + for (const [key, value] of Object.entries(object)) { + traverse(value, [...path, key]) + } + } + } + traverse(frontMatter) + return Array.from(keyValues).sort() +} + export async function getSpecialPage( ctx: ContentsgartenRequestContext, pageRef: string, diff --git a/packages/contentsgarten/src/testing.ts b/packages/contentsgarten/src/testing.ts index ddd78d1f..887ca0f8 100644 --- a/packages/contentsgarten/src/testing.ts +++ b/packages/contentsgarten/src/testing.ts @@ -75,6 +75,15 @@ export namespace testing { lastModified: a.lastModified, })) }, + async queryPages() { + return { + count: 0, + results: [], + } + }, + async checkTypo() { + return [] + }, } }