Skip to content

Commit

Permalink
src: fix symlink issue, subtitution middleware hack
Browse files Browse the repository at this point in the history
Fixes #163
  • Loading branch information
flakey5 committed Dec 21, 2024
1 parent 8bba5bb commit 70f9e7a
Show file tree
Hide file tree
Showing 16 changed files with 2,589 additions and 242 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/update-links.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
run: npm install && npm update nodejs-latest-linker --save

- name: Update Redirect Links
run: node scripts/update-latest-versions.js && npm run format
run: node scripts/build-r2-symlinks.mjs && npm run format
env:
CF_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
CF_SECRET_ACCESS_KEY: ${{ secrets.CF_SECRET_ACCESS_KEY }}
Expand Down
270 changes: 270 additions & 0 deletions scripts/build-r2-symlinks.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
#!/usr/bin/env bash
'use strict';

import { join } from 'node:path';
import { writeFile } from 'node:fs/promises';
import {
HeadObjectCommand,
ListObjectsV2Command,
S3Client,
} from '@aws-sdk/client-s3';
import { Linker } from 'nodejs-latest-linker/common.js';
import { DOCS_DIR, ENDPOINT, PROD_BUCKET, RELEASE_DIR } from './constants.mjs';

const DOCS_DIRECTORY_OUT = join(
import.meta.dirname,
'..',
'src',
'constants',
'docsDirectory.json'
);

const LATEST_VERSIONS_OUT = join(
import.meta.dirname,
'..',
'src',
'constants',
'latestVersions.json'
);

const CACHED_DIRECTORIES_OUT = join(
import.meta.dirname,
'..',
'src',
'constants',
'cachedDirectories.json'
);

if (!process.env.CF_ACCESS_KEY_ID) {
throw new TypeError('CF_ACCESS_KEY_ID missing');
}

if (!process.env.CF_ACCESS_KEY_ID) {
throw new TypeError('CF_SECRET_ACCESS_KEY missing');
}

const client = new S3Client({
endpoint: ENDPOINT,
region: 'auto',
credentials: {
accessKeyId: process.env.CF_ACCESS_KEY_ID,
secretAccessKey: process.env.CF_SECRET_ACCESS_KEY,
},
});

// Cache the contents of `nodejs/docs/` so we can reference it in the worker
await writeDocsDirectoryFile(client);

// Grab all of the files & directories in `nodejs/release/`
const releases = await listDirectory(client, RELEASE_DIR);

// Create the latest version mapping with the contents of `nodejs/release/`
const latestVersions = await getLatestVersionMapping(client, releases);

// Write it so we can use it in the worker
await writeFile(LATEST_VERSIONS_OUT, JSON.stringify(latestVersions));

// Filter the latest version map so we only have the `latest-*` directories
const latestVersionDirectories = Object.keys(latestVersions).map(version =>
version === 'node-latest.tar.gz' ? version : `${version}/`
);

// Create the complete listing of `nodejs/release/` by adding what R2 returned
// and the latest version directories (which are the symlinks)
const releaseDirectorySubdirectories = releases.subdirectories
.concat(latestVersionDirectories)
.sort();

// This is the path in R2 for the latest tar archive of Node.
const nodeLatestPath = `nodejs/release/${latestVersions['node-latest.tar.gz'].replaceAll('latest', latestVersions['latest'])}`;

// Stat the file that `node-latest.tar.gz` points to so we can have accurate
// size & last modified info for the directory listing
const nodeLatest = await headFile(client, nodeLatestPath);
if (!nodeLatest) {
throw new TypeError(
`node-latest.tar.gz points to ${latestVersions['node-latest.tar.gz']} which doesn't exist in the prod bucket`
);
}
/**
* Preprocess these directories since they have symlinks in them that don't
* actually exist in R2 but need to be present when we give a directory listing
* result
* @type {Record<string, import('../src/providers/provider.ts').ReadDirectoryResult>}
*/
const cachedDirectories = {
'nodejs/release/': {
subdirectories: releaseDirectorySubdirectories,
hasIndexHtmlFile: false,
files: [
...releases.files,
{
name: 'node-latest.tar.gz',
lastModified: nodeLatest.lastModified,
size: nodeLatest.size,
},
],
lastModified: releases.lastModified,
},
'nodejs/docs/': {
// We reuse the releases listing result here instead of listing the docs
// directory since it's more complete. The docs folder does have some actual
// directories in it, but most of it is symlinks and aren't present in R2.
subdirectories: releaseDirectorySubdirectories,
hasIndexHtmlFile: false,
files: [],
lastModified: releases.lastModified,
},
};
await writeFile(CACHED_DIRECTORIES_OUT, JSON.stringify(cachedDirectories));
/**
* @param {S3Client} client
* @param {string} directory
* @returns {Promise<import('../src/providers/provider.js').ReadDirectoryResult>}
*/
async function listDirectory(client, directory) {
/**
* @type {Array<string>}
*/
const subdirectories = [];
/**
* @type {Array<import('../src/providers/provider.js').File>}
*/
const files = [];
let hasIndexHtmlFile = false;
let truncated = true;
let continuationToken;
let lastModified = new Date(0);
while (truncated) {
const data = await client.send(
new ListObjectsV2Command({
Bucket: PROD_BUCKET,
Delimiter: '/',
Prefix: directory,
ContinuationToken: continuationToken,
})
);
if (data.CommonPrefixes) {
data.CommonPrefixes.forEach(value => {
if (value.Prefix) {
subdirectories.push(value.Prefix.substring(directory.length));
}
});
}
if (data.Contents) {
data.Contents.forEach(value => {
if (value.Key) {
if (value.Key.match(/index.htm(?:l)$/)) {
hasIndexHtmlFile = true;
}
files.push({
name: value.Key.substring(directory.length),
lastModified: value.LastModified,
size: value.Size,
});
if (value.LastModified > lastModified) {
lastModified = value.LastModified;
}
}
});
}
truncated = data.IsTruncated;
continuationToken = data.NextContinuationToken;
}
return { subdirectories, hasIndexHtmlFile, files, lastModified };
}
/**
* @param {S3Client} client
* @param {string} path
* @returns {Promise<import('../src/providers/provider.js').File | undefined>}
*/
async function headFile(client, path) {
const data = await client.send(
new HeadObjectCommand({
Bucket: PROD_BUCKET,
Key: path,
})
);
if (!data.LastModified || !data.ContentLength) {
return undefined;
}
return {
name: path,
lastModified: data.LastModified,
size: data.ContentLength,
};
}
/**
* @param {S3Client} client
*/
async function writeDocsDirectoryFile(client) {
// Grab all of the directories in `nodejs/docs/`
const docs = await listDirectory(client, DOCS_DIR);
// Cache the contents of `nodejs/docs/` so we can refer to it in the worker w/o
// making a call to R2.
await writeFile(
DOCS_DIRECTORY_OUT,
JSON.stringify(
docs.subdirectories.map(subdirectory =>
subdirectory.substring(0, subdirectory.length - 1)
)
)
);
}
/**
* @param {S3Client} client
* @param {import('../src/providers/provider.js').ReadDirectoryResult} releases
* @returns {Promise<Record<string, string>>}
*/
async function getLatestVersionMapping(client, releases) {
const linker = new Linker({ baseDir: RELEASE_DIR, docs: DOCS_DIR });
/**
* Creates mappings to the latest versions of Node
* @type {Map<string, string>}
* @example { 'nodejs/release/latest-v20.x': 'nodejs/release/v20.x.x' }
*/
const links = await linker.getLinks(
[...releases.subdirectories, ...releases.files.map(file => file.name)],
async directory => {
const { subdirectories, files } = await listDirectory(
client,
`${directory}/`
);
return [...subdirectories, ...files.map(file => file.name)];
}
);
/**
* @type {Record<string, string>}
* @example {'latest-v20.x': 'v20.x.x'}
*/
const latestVersions = {};
for (const [key, value] of links) {
const trimmedKey = key.substring(RELEASE_DIR.length);
const trimmedValue = value.substring(RELEASE_DIR.length);
latestVersions[trimmedKey] = trimmedValue;
}
return latestVersions;
}
File renamed without changes.
4 changes: 4 additions & 0 deletions scripts/constants.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,7 @@ export const PROD_BUCKET = process.env.PROD_BUCKET ?? 'dist-prod';
export const STAGING_BUCKET = process.env.STAGING_BUCKET ?? 'dist-staging';

export const R2_RETRY_COUNT = 3;

export const RELEASE_DIR = 'nodejs/release/';

export const DOCS_DIR = 'nodejs/docs/';
Loading

0 comments on commit 70f9e7a

Please sign in to comment.