Skip to content

Commit

Permalink
Add S3 as DBDocsDefinition cache for revalidate-all/loadWithUrl/finis…
Browse files Browse the repository at this point in the history 
…hRegister paths (#2048)
  • Loading branch information
@dubwub
dubwub authored Jan 22, 2025
1 parent 1db1d7a commit 00348c9
Show file tree
Hide file tree
Showing 7 changed files with 166 additions and 6 deletions.
27 changes: 21 additions & 6 deletions packages/fern-docs/bundle/src/server/DocsLoader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,12 +123,27 @@ export class DocsLoader {
DocsV2Read.LoadDocsForUrlResponse | undefined
> {
if (!this.#loadForDocsUrlResponse) {
const response = await loadWithUrl(this.domain);

if (response.ok) {
this.#loadForDocsUrlResponse = response.body;
} else {
this.#error = response.error;
try {
const environmentType = process.env.NODE_ENV ?? "development";
let dbDocsDefUrl = "";
if (environmentType === "development") {
dbDocsDefUrl = `https://docs-definitions-dev2.buildwithfern.com/${this.domain}.json`;
} else if (environmentType === "production") {
dbDocsDefUrl = `https://docs-definitions.buildwithfern.com/${this.domain}.json`;
}
const response = await fetch(dbDocsDefUrl);
if (response.ok) {
const json = await response.json();
return json as DocsV2Read.LoadDocsForUrlResponse;
}
} catch {
// Not served by cloudfront, fetch from Redis and then RDS
const response = await loadWithUrl(this.domain);
if (response.ok) {
this.#loadForDocsUrlResponse = response.body;
} else {
this.#error = response.error;
}
}
}
return this.#loadForDocsUrlResponse;
Expand Down
58 changes: 58 additions & 0 deletions servers/fdr-deploy/scripts/fdr-deploy-stack.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,53 @@ export class FdrDeployStack extends Stack {
}
);

// for revalidate-all and finish-register workflow
const dbDocsDefinitionBucket = new Bucket(
this,
"fdr-docs-definitions-public",
{
bucketName: `fdr-${environmentType.toLowerCase()}-docs-definitions-public`,
cors: [
{
allowedMethods: [
HttpMethods.GET,
HttpMethods.POST,
HttpMethods.PUT,
],
allowedOrigins: ["*"],
allowedHeaders: ["*"],
},
],
blockPublicAccess: {
blockPublicAcls: false,
blockPublicPolicy: false,
ignorePublicAcls: false,
restrictPublicBuckets: false,
},
versioned: true,
}
);
dbDocsDefinitionBucket.grantPublicAccess();

const dbDocsDefinitionDomainName =
environmentType === "PROD"
? "docs-definitions.buildwithfern.com"
: "docs-definitions-dev2.buildwithfern.com";
const dbDocsDefinitionDistribution = new cloudfront.Distribution(
this,
"DbDocsDefinitionDistribution",
{
defaultBehavior: {
origin: new origins.S3Origin(dbDocsDefinitionBucket),
viewerProtocolPolicy:
cloudfront.ViewerProtocolPolicy.REDIRECT_TO_HTTPS,
cachePolicy: cloudfront.CachePolicy.CACHING_OPTIMIZED,
},
domainNames: [dbDocsDefinitionDomainName],
certificate,
}
);

new route53.ARecord(this, "PublicDocsFilesRecord", {
recordName: publicDocsFilesDomainName,
target: route53.RecordTarget.fromAlias(
Expand All @@ -215,6 +262,14 @@ export class FdrDeployStack extends Stack {
zone: hostedZone,
});

new route53.ARecord(this, "DbDocsDefinitionRecord", {
recordName: dbDocsDefinitionDomainName,
target: route53.RecordTarget.fromAlias(
new targets.CloudFrontTarget(dbDocsDefinitionDistribution)
),
zone: hostedZone,
});

const fernDocsCacheEndpoint = this.constructElastiCacheInstance(this, {
cacheName: options.cacheName,
IVpc: vpc,
Expand Down Expand Up @@ -265,6 +320,9 @@ export class FdrDeployStack extends Stack {
PUBLIC_S3_BUCKET_REGION: publicDocsBucket.stack.region,
PRIVATE_S3_BUCKET_NAME: privateDocsBucket.bucketName,
PRIVATE_S3_BUCKET_REGION: privateDocsBucket.stack.region,
DB_DOCS_DEFINITION_BUCKET_NAME: dbDocsDefinitionBucket.bucketName,
DB_DOCS_DEFINITION_BUCKET_REGION:
dbDocsDefinitionBucket.stack.region,
API_DEFINITION_SOURCE_BUCKET_NAME:
privateApiDefinitionSourceBucket.bucketName,
API_DEFINITION_SOURCE_BUCKET_REGION:
Expand Down
5 changes: 5 additions & 0 deletions servers/fdr/src/__test__/local/s3.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ describe("S3 Service", () => {
bucketRegion: "us-east-1",
urlOverride: undefined,
},
dbDocsDefinitionS3: {
bucketName: "fdr-dev2-db-docs-def-public",
bucketRegion: "us-east-1",
urlOverride: undefined,
},
privateApiDefinitionSourceS3: {
bucketName: "fdr-source-files",
bucketRegion: "us-east-1",
Expand Down
5 changes: 5 additions & 0 deletions servers/fdr/src/__test__/mock.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,11 @@ export const baseMockFdrConfig: FdrConfig = {
bucketRegion: "us-east-1",
urlOverride: "http://s3-mock:9090",
},
dbDocsDefinitionS3: {
bucketName: "fdr",
bucketRegion: "us-east-1",
urlOverride: "http://s3-mock:9090",
},
privateApiDefinitionSourceS3: {
bucketName: "fdr",
bucketRegion: "us-east-1",
Expand Down
16 changes: 16 additions & 0 deletions servers/fdr/src/app/FdrConfig.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ const PRIVATE_S3_BUCKET_NAME_ENV_VAR = "PRIVATE_S3_BUCKET_NAME";
const PRIVATE_S3_BUCKET_REGION_ENV_VAR = "PRIVATE_S3_BUCKET_REGION";
const PRIVATE_S3_URL_OVERRIDE_ENV_VAR = "PRIVATE_S3_URL_OVERRIDE";

const DB_DOCS_DEFINITION_BUCKET_NAME_ENV_VAR = "DB_DOCS_DEFINITION_BUCKET_NAME";
const DB_DOCS_DEFINITION_BUCKET_REGION_ENV_VAR =
"DB_DOCS_DEFINITION_BUCKET_REGION";
const DB_DOCS_DEFINITION_BUCKET_URL_OVERRIDE_ENV_VAR =
"DB_DOCS_DEFINITION_BUCKET_URL_OVERRIDE";

const API_DEFINITION_SOURCE_BUCKET_NAME_ENV_VAR =
"API_DEFINITION_SOURCE_BUCKET_NAME";
const API_DEFINITION_SOURCE_BUCKET_REGION_ENV_VAR =
Expand Down Expand Up @@ -45,6 +51,7 @@ export interface FdrConfig {
cdnPublicDocsUrl: string;
publicDocsS3: S3Config;
privateDocsS3: S3Config;
dbDocsDefinitionS3: S3Config;
privateApiDefinitionSourceS3: S3Config;
domainSuffix: string;
algoliaAppId: string;
Expand Down Expand Up @@ -80,6 +87,15 @@ export function getConfig(): FdrConfig {
),
urlOverride: process.env[PRIVATE_S3_URL_OVERRIDE_ENV_VAR],
},
dbDocsDefinitionS3: {
bucketName: getEnvironmentVariableOrThrow(
DB_DOCS_DEFINITION_BUCKET_NAME_ENV_VAR
),
bucketRegion: getEnvironmentVariableOrThrow(
DB_DOCS_DEFINITION_BUCKET_REGION_ENV_VAR
),
urlOverride: process.env[DB_DOCS_DEFINITION_BUCKET_URL_OVERRIDE_ENV_VAR],
},
privateApiDefinitionSourceS3: {
bucketName: getEnvironmentVariableOrThrow(
API_DEFINITION_SOURCE_BUCKET_NAME_ENV_VAR
Expand Down
30 changes: 30 additions & 0 deletions servers/fdr/src/controllers/docs/v2/getDocsWriteV2Service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,36 @@ export function getDocsWriteV2Service(app: FdrApplication): DocsV2WriteService {
indexSegments,
});

const readDocsDefinition = convertDocsDefinitionToRead({
docsDbDefinition: dbDocsDefinition,
algoliaSearchIndex: undefined,
filesV2: {},
apis: mapValues(apiDefinitionsById, (def) =>
convertDbAPIDefinitionToRead(def)
),
apisV2: mapValues(apiDefinitionsLatestById, (def) => def),
id: DocsV1Write.DocsConfigId(""),
search: getSearchInfoFromDocs({
algoliaIndex: undefined,
indexSegmentIds: [],
activeIndexSegments: [],
docsDbDefinition: dbDocsDefinition,
app,
}),
});

try {
await app.services.s3.writeDBDocsDefinition({
domain: docsRegistrationInfo.fernUrl.getFullUrl(),
readDocsDefinition,
});
} catch (e) {
app.logger.error(
`Error while trying to write DB docs definition for ${docsRegistrationInfo.fernUrl}`,
e
);
}

/**
* IMPORTANT NOTE:
* vercel cache is not shared between custom domains, so we need to revalidate on EACH custom domain individually
Expand Down
31 changes: 31 additions & 0 deletions servers/fdr/src/services/s3/S3Service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
GetObjectCommand,
PutObjectCommand,
PutObjectCommandInput,
PutObjectCommandOutput,
S3Client,
} from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
Expand Down Expand Up @@ -37,6 +38,10 @@ export interface S3ApiDefinitionSourceFileInfo {
}

export interface S3Service {
writeDBDocsDefinition(arg0: {
domain: string;
readDocsDefinition: any;
}): Promise<PutObjectCommandOutput>;
getPresignedDocsAssetsUploadUrls({
domain,
filepaths,
Expand Down Expand Up @@ -79,6 +84,7 @@ export class S3ServiceImpl implements S3Service {
private publicDocsS3: S3Client;
private privateDocsS3: S3Client;
private privateApiDefinitionSourceS3: S3Client;
private dbDocsDefinitionS3: S3Client;
private presignedDownloadUrlCache = new Cache<string>(
10_000,
ONE_WEEK_IN_SECONDS
Expand Down Expand Up @@ -106,6 +112,16 @@ export class S3ServiceImpl implements S3Service {
secretAccessKey: config.awsSecretKey,
},
});
this.dbDocsDefinitionS3 = new S3Client({
...(config.dbDocsDefinitionS3.urlOverride != null
? { endpoint: config.dbDocsDefinitionS3.urlOverride }
: {}),
region: config.dbDocsDefinitionS3.bucketRegion,
credentials: {
accessKeyId: config.awsAccessKey,
secretAccessKey: config.awsSecretKey,
},
});
this.privateApiDefinitionSourceS3 = new S3Client({
...(config.privateApiDefinitionSourceS3.urlOverride != null
? { endpoint: config.privateApiDefinitionSourceS3.urlOverride }
Expand Down Expand Up @@ -315,6 +331,21 @@ export class S3ServiceImpl implements S3Service {
};
}

async writeDBDocsDefinition({
domain,
readDocsDefinition,
}: {
domain: string;
readDocsDefinition: any;
}): Promise<PutObjectCommandOutput> {
const command = new PutObjectCommand({
Bucket: this.config.dbDocsDefinitionS3.bucketName,
Key: `${domain}.json`,
Body: JSON.stringify(readDocsDefinition),
});
return await this.dbDocsDefinitionS3.send(command);
}

constructS3DocsKey({
domain,
time,
Expand Down

0 comments on commit 00348c9

Please sign in to comment.