From 00348c9cc8586b48780f59f0688540f098684069 Mon Sep 17 00:00:00 2001
From: Darwin Ding <dingdarwin@gmail.com>
Date: Wed, 22 Jan 2025 14:55:14 -0500
Subject: [PATCH] Add S3 as DBDocsDefinition cache for
revalidate-all/loadWithUrl/finishRegister paths (#2048)
---
.../fern-docs/bundle/src/server/DocsLoader.ts | 27 +++++++--
.../fdr-deploy/scripts/fdr-deploy-stack.ts | 58 +++++++++++++++++++
servers/fdr/src/__test__/local/s3.test.ts | 5 ++
servers/fdr/src/__test__/mock.ts | 5 ++
servers/fdr/src/app/FdrConfig.ts | 16 +++++
.../docs/v2/getDocsWriteV2Service.ts | 30 ++++++++++
servers/fdr/src/services/s3/S3Service.ts | 31 ++++++++++
7 files changed, 166 insertions(+), 6 deletions(-)
diff --git a/packages/fern-docs/bundle/src/server/DocsLoader.ts b/packages/fern-docs/bundle/src/server/DocsLoader.ts
index 8dd91b034e..dac3e8b8b2 100644
--- a/packages/fern-docs/bundle/src/server/DocsLoader.ts
+++ b/packages/fern-docs/bundle/src/server/DocsLoader.ts
@@ -123,12 +123,27 @@ export class DocsLoader {
DocsV2Read.LoadDocsForUrlResponse | undefined
> {
if (!this.#loadForDocsUrlResponse) {
- const response = await loadWithUrl(this.domain);
-
- if (response.ok) {
- this.#loadForDocsUrlResponse = response.body;
- } else {
- this.#error = response.error;
+ try {
+ const environmentType = process.env.NODE_ENV ?? "development";
+ let dbDocsDefUrl = "";
+ if (environmentType === "development") {
+ dbDocsDefUrl = `https://docs-definitions-dev2.buildwithfern.com/${this.domain}.json`;
+ } else if (environmentType === "production") {
+ dbDocsDefUrl = `https://docs-definitions.buildwithfern.com/${this.domain}.json`;
+ }
+ const response = await fetch(dbDocsDefUrl);
+ if (response.ok) {
+ const json = await response.json();
+ return json as DocsV2Read.LoadDocsForUrlResponse;
+ }
+ } catch {
+ // Not served by cloudfront, fetch from Redis and then RDS
+ const response = await loadWithUrl(this.domain);
+ if (response.ok) {
+ this.#loadForDocsUrlResponse = response.body;
+ } else {
+ this.#error = response.error;
+ }
}
}
return this.#loadForDocsUrlResponse;
diff --git a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts
index 0fd3e1f965..95aa266aa1 100644
--- a/servers/fdr-deploy/scripts/fdr-deploy-stack.ts
+++ b/servers/fdr-deploy/scripts/fdr-deploy-stack.ts
@@ -207,6 +207,53 @@ export class FdrDeployStack extends Stack {
}
);
+ // for revalidate-all and finish-register workflow
+ const dbDocsDefinitionBucket = new Bucket(
+ this,
+ "fdr-docs-definitions-public",
+ {
+ bucketName: `fdr-${environmentType.toLowerCase()}-docs-definitions-public`,
+ cors: [
+ {
+ allowedMethods: [
+ HttpMethods.GET,
+ HttpMethods.POST,
+ HttpMethods.PUT,
+ ],
+ allowedOrigins: ["*"],
+ allowedHeaders: ["*"],
+ },
+ ],
+ blockPublicAccess: {
+ blockPublicAcls: false,
+ blockPublicPolicy: false,
+ ignorePublicAcls: false,
+ restrictPublicBuckets: false,
+ },
+ versioned: true,
+ }
+ );
+ dbDocsDefinitionBucket.grantPublicAccess();
+
+ const dbDocsDefinitionDomainName =
+ environmentType === "PROD"
+ ? "docs-definitions.buildwithfern.com"
+ : "docs-definitions-dev2.buildwithfern.com";
+ const dbDocsDefinitionDistribution = new cloudfront.Distribution(
+ this,
+ "DbDocsDefinitionDistribution",
+ {
+ defaultBehavior: {
+ origin: new origins.S3Origin(dbDocsDefinitionBucket),
+ viewerProtocolPolicy:
+ cloudfront.ViewerProtocolPolicy.REDIRECT_TO_HTTPS,
+ cachePolicy: cloudfront.CachePolicy.CACHING_OPTIMIZED,
+ },
+ domainNames: [dbDocsDefinitionDomainName],
+ certificate,
+ }
+ );
+
new route53.ARecord(this, "PublicDocsFilesRecord", {
recordName: publicDocsFilesDomainName,
target: route53.RecordTarget.fromAlias(
@@ -215,6 +262,14 @@ export class FdrDeployStack extends Stack {
zone: hostedZone,
});
+ new route53.ARecord(this, "DbDocsDefinitionRecord", {
+ recordName: dbDocsDefinitionDomainName,
+ target: route53.RecordTarget.fromAlias(
+ new targets.CloudFrontTarget(dbDocsDefinitionDistribution)
+ ),
+ zone: hostedZone,
+ });
+
const fernDocsCacheEndpoint = this.constructElastiCacheInstance(this, {
cacheName: options.cacheName,
IVpc: vpc,
@@ -265,6 +320,9 @@ export class FdrDeployStack extends Stack {
PUBLIC_S3_BUCKET_REGION: publicDocsBucket.stack.region,
PRIVATE_S3_BUCKET_NAME: privateDocsBucket.bucketName,
PRIVATE_S3_BUCKET_REGION: privateDocsBucket.stack.region,
+ DB_DOCS_DEFINITION_BUCKET_NAME: dbDocsDefinitionBucket.bucketName,
+ DB_DOCS_DEFINITION_BUCKET_REGION:
+ dbDocsDefinitionBucket.stack.region,
API_DEFINITION_SOURCE_BUCKET_NAME:
privateApiDefinitionSourceBucket.bucketName,
API_DEFINITION_SOURCE_BUCKET_REGION:
diff --git a/servers/fdr/src/__test__/local/s3.test.ts b/servers/fdr/src/__test__/local/s3.test.ts
index 70d985e486..c9bc3d6b06 100644
--- a/servers/fdr/src/__test__/local/s3.test.ts
+++ b/servers/fdr/src/__test__/local/s3.test.ts
@@ -19,6 +19,11 @@ describe("S3 Service", () => {
bucketRegion: "us-east-1",
urlOverride: undefined,
},
+ dbDocsDefinitionS3: {
+ bucketName: "fdr-dev2-db-docs-def-public",
+ bucketRegion: "us-east-1",
+ urlOverride: undefined,
+ },
privateApiDefinitionSourceS3: {
bucketName: "fdr-source-files",
bucketRegion: "us-east-1",
diff --git a/servers/fdr/src/__test__/mock.ts b/servers/fdr/src/__test__/mock.ts
index 866e685d3a..ae68844b22 100644
--- a/servers/fdr/src/__test__/mock.ts
+++ b/servers/fdr/src/__test__/mock.ts
@@ -150,6 +150,11 @@ export const baseMockFdrConfig: FdrConfig = {
bucketRegion: "us-east-1",
urlOverride: "http://s3-mock:9090",
},
+ dbDocsDefinitionS3: {
+ bucketName: "fdr",
+ bucketRegion: "us-east-1",
+ urlOverride: "http://s3-mock:9090",
+ },
privateApiDefinitionSourceS3: {
bucketName: "fdr",
bucketRegion: "us-east-1",
diff --git a/servers/fdr/src/app/FdrConfig.ts b/servers/fdr/src/app/FdrConfig.ts
index c083cf325e..60d2401654 100644
--- a/servers/fdr/src/app/FdrConfig.ts
+++ b/servers/fdr/src/app/FdrConfig.ts
@@ -10,6 +10,12 @@ const PRIVATE_S3_BUCKET_NAME_ENV_VAR = "PRIVATE_S3_BUCKET_NAME";
const PRIVATE_S3_BUCKET_REGION_ENV_VAR = "PRIVATE_S3_BUCKET_REGION";
const PRIVATE_S3_URL_OVERRIDE_ENV_VAR = "PRIVATE_S3_URL_OVERRIDE";
+const DB_DOCS_DEFINITION_BUCKET_NAME_ENV_VAR = "DB_DOCS_DEFINITION_BUCKET_NAME";
+const DB_DOCS_DEFINITION_BUCKET_REGION_ENV_VAR =
+ "DB_DOCS_DEFINITION_BUCKET_REGION";
+const DB_DOCS_DEFINITION_BUCKET_URL_OVERRIDE_ENV_VAR =
+ "DB_DOCS_DEFINITION_BUCKET_URL_OVERRIDE";
+
const API_DEFINITION_SOURCE_BUCKET_NAME_ENV_VAR =
"API_DEFINITION_SOURCE_BUCKET_NAME";
const API_DEFINITION_SOURCE_BUCKET_REGION_ENV_VAR =
@@ -45,6 +51,7 @@ export interface FdrConfig {
cdnPublicDocsUrl: string;
publicDocsS3: S3Config;
privateDocsS3: S3Config;
+ dbDocsDefinitionS3: S3Config;
privateApiDefinitionSourceS3: S3Config;
domainSuffix: string;
algoliaAppId: string;
@@ -80,6 +87,15 @@ export function getConfig(): FdrConfig {
),
urlOverride: process.env[PRIVATE_S3_URL_OVERRIDE_ENV_VAR],
},
+ dbDocsDefinitionS3: {
+ bucketName: getEnvironmentVariableOrThrow(
+ DB_DOCS_DEFINITION_BUCKET_NAME_ENV_VAR
+ ),
+ bucketRegion: getEnvironmentVariableOrThrow(
+ DB_DOCS_DEFINITION_BUCKET_REGION_ENV_VAR
+ ),
+ urlOverride: process.env[DB_DOCS_DEFINITION_BUCKET_URL_OVERRIDE_ENV_VAR],
+ },
privateApiDefinitionSourceS3: {
bucketName: getEnvironmentVariableOrThrow(
API_DEFINITION_SOURCE_BUCKET_NAME_ENV_VAR
diff --git a/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts b/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts
index a24a8addd8..c3d4136fda 100644
--- a/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts
+++ b/servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts
@@ -277,6 +277,36 @@ export function getDocsWriteV2Service(app: FdrApplication): DocsV2WriteService {
indexSegments,
});
+ const readDocsDefinition = convertDocsDefinitionToRead({
+ docsDbDefinition: dbDocsDefinition,
+ algoliaSearchIndex: undefined,
+ filesV2: {},
+ apis: mapValues(apiDefinitionsById, (def) =>
+ convertDbAPIDefinitionToRead(def)
+ ),
+ apisV2: mapValues(apiDefinitionsLatestById, (def) => def),
+ id: DocsV1Write.DocsConfigId(""),
+ search: getSearchInfoFromDocs({
+ algoliaIndex: undefined,
+ indexSegmentIds: [],
+ activeIndexSegments: [],
+ docsDbDefinition: dbDocsDefinition,
+ app,
+ }),
+ });
+
+ try {
+ await app.services.s3.writeDBDocsDefinition({
+ domain: docsRegistrationInfo.fernUrl.getFullUrl(),
+ readDocsDefinition,
+ });
+ } catch (e) {
+ app.logger.error(
+ `Error while trying to write DB docs definition for ${docsRegistrationInfo.fernUrl}`,
+ e
+ );
+ }
+
/**
* IMPORTANT NOTE:
* vercel cache is not shared between custom domains, so we need to revalidate on EACH custom domain individually
diff --git a/servers/fdr/src/services/s3/S3Service.ts b/servers/fdr/src/services/s3/S3Service.ts
index 5d6015645b..474f85ecf7 100644
--- a/servers/fdr/src/services/s3/S3Service.ts
+++ b/servers/fdr/src/services/s3/S3Service.ts
@@ -2,6 +2,7 @@ import {
GetObjectCommand,
PutObjectCommand,
PutObjectCommandInput,
+ PutObjectCommandOutput,
S3Client,
} from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
@@ -37,6 +38,10 @@ export interface S3ApiDefinitionSourceFileInfo {
}
export interface S3Service {
+ writeDBDocsDefinition(arg0: {
+ domain: string;
+ readDocsDefinition: any;
+ }): Promise<PutObjectCommandOutput>;
getPresignedDocsAssetsUploadUrls({
domain,
filepaths,
@@ -79,6 +84,7 @@ export class S3ServiceImpl implements S3Service {
private publicDocsS3: S3Client;
private privateDocsS3: S3Client;
private privateApiDefinitionSourceS3: S3Client;
+ private dbDocsDefinitionS3: S3Client;
private presignedDownloadUrlCache = new Cache<string>(
10_000,
ONE_WEEK_IN_SECONDS
@@ -106,6 +112,16 @@ export class S3ServiceImpl implements S3Service {
secretAccessKey: config.awsSecretKey,
},
});
+ this.dbDocsDefinitionS3 = new S3Client({
+ ...(config.dbDocsDefinitionS3.urlOverride != null
+ ? { endpoint: config.dbDocsDefinitionS3.urlOverride }
+ : {}),
+ region: config.dbDocsDefinitionS3.bucketRegion,
+ credentials: {
+ accessKeyId: config.awsAccessKey,
+ secretAccessKey: config.awsSecretKey,
+ },
+ });
this.privateApiDefinitionSourceS3 = new S3Client({
...(config.privateApiDefinitionSourceS3.urlOverride != null
? { endpoint: config.privateApiDefinitionSourceS3.urlOverride }
@@ -315,6 +331,21 @@ export class S3ServiceImpl implements S3Service {
};
}
+ async writeDBDocsDefinition({
+ domain,
+ readDocsDefinition,
+ }: {
+ domain: string;
+ readDocsDefinition: any;
+ }): Promise<PutObjectCommandOutput> {
+ const command = new PutObjectCommand({
+ Bucket: this.config.dbDocsDefinitionS3.bucketName,
+ Key: `${domain}.json`,
+ Body: JSON.stringify(readDocsDefinition),
+ });
+ return await this.dbDocsDefinitionS3.send(command);
+ }
+
constructS3DocsKey({
domain,
time,