src, scripts: better directory cache

.env.example

@@ -1,30 +1,30 @@
 # Your Cloudflare account tag.
-# 
+#
 # Needed for:
 #  - Directory cache scripts
 CLOUDFLARE_ACCOUNT_ID=
 
 # Cloudflare V4 API token.
-# 
+#
 # Needed for:
 #  - Directory cache scripts
-# 
+#
 # Required permissions:
 #  - `Workers KV Storage`: Edit
 #  - `Workers R2 Storage`: Read
-# 
+#
 # See https://developers.cloudflare.com/fundamentals/api/get-started/create-token/
 CLOUDFLARE_API_TOKEN=
 
 # S3 credentials for your R2 bucket.
-# 
+#
 # Needed for:
 #  - Directory listings in the worker.
 #  - Directory cache scripts
-# 
+#
 # Required permissions:
 #  - `Object Read Only`
-# 
+#
 # See https://dash.cloudflare.com/?account=/r2/api-tokens
 S3_ACCESS_KEY_ID=
 S3_ACCESS_KEY_SECRET=

.github/workflows/update-links.yml

@@ -7,12 +7,11 @@ on:
   # Triggered by https://github.com/nodejs/node/blob/main/.github/workflows/update-release-links.yml
   workflow_dispatch:
     inputs:
+      # Not used but kept around to not break Node core ci since it passes this
       version:
         description: 'Node.js version (ex/ `v20.0.0`)'
-        required: true
+        required: false
         type: string
-  schedule:
-    - cron: '0 0 * * *'
 
 concurrency:
   group: update-redirect-links
@@ -60,7 +59,7 @@ jobs:
         run: npm ci && npm update nodejs-latest-linker --save
 
       - name: Update Directory Cache
-        run: node scripts/update-directory-cache.mjs "$VERSION_INPUT" && node --run format
+        run: node scripts/handle-mainline-release.mjs && node --run format
         env:
           VERSION_INPUT: "${{ inputs.version || '' }}"
           CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}

e2e-tests/directory.kv.test.ts

@@ -1,5 +1,5 @@
-import { env, createExecutionContext } from 'cloudflare:test';
-import { test, beforeAll, expect, vi } from 'vitest';
+import { env, fetchMock, createExecutionContext } from 'cloudflare:test';
+import { test, beforeAll, expect, vi, afterEach } from 'vitest';
 import {
   populateDirectoryCacheWithDevBucket,
   populateR2WithDevBucket,
@@ -13,9 +13,46 @@ const mockedEnv: Env = {
   ENVIRONMENT: 'e2e-tests',
   LOG_ERRORS: true,
   USE_KV: true,
+  S3_ENDPOINT: 'https://s3.mock',
+  S3_ACCESS_KEY_ID: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+  S3_ACCESS_KEY_SECRET:
+    'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
 };
 
+const s3Url = new URL(mockedEnv.S3_ENDPOINT);
+s3Url.host = `${mockedEnv.BUCKET_NAME}.${s3Url.host}`;
+
+const S3_DIRECTORY_RESULT = `
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
+  <Name>dist-prod</Name>
+  <Prefix />
+  <Marker />
+  <MaxKeys>1000</MaxKeys>
+  <IsTruncated>false</IsTruncated>
+  <CommonPrefixes>
+    <Prefix>nodejs/release/v1.0.0/latest/</Prefix>
+  </CommonPrefixes>
+  <Contents>
+    <ETag>"asd123"</ETag>
+    <Key>nodejs/release/v1.0.0/index.json</Key>
+    <LastModified>2023-09-12T05:43:00.000Z</LastModified>
+    <Size>18</Size>
+  </Contents>
+</ListBucketResult>`;
+
+const S3_EMPTY_DIRECTORY_RESULT = `
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
+  <Name>dist-prod</Name>
+  <Prefix />
+  <Marker />
+  <MaxKeys>1000</MaxKeys>
+  <IsTruncated>false</IsTruncated>
+</ListBucketResult>`;
+
 beforeAll(async () => {
+  fetchMock.activate();
+  fetchMock.disableNetConnect();
+
   await populateR2WithDevBucket();
   await populateDirectoryCacheWithDevBucket();
 
@@ -42,6 +79,10 @@ beforeAll(async () => {
   );
 });
 
+afterEach(() => {
+  fetchMock.assertNoPendingInterceptors();
+});
+
 // Ensure essential endpoints are routable
 for (const path of ['/dist/', '/docs/', '/api/', '/download/', '/metrics/']) {
   test(`GET \`${path}\` returns 200`, async () => {
@@ -61,6 +102,16 @@ for (const path of ['/dist/', '/docs/', '/api/', '/download/', '/metrics/']) {
 }
 
 test('GET `/dist/unknown-directory/` returns 404', async () => {
+  fetchMock
+    .get(s3Url.origin)
+    .intercept({
+      path: /.*/,
+      query: {
+        prefix: 'nodejs/release/unknown-directory/',
+      },
+    })
+    .reply(200, S3_EMPTY_DIRECTORY_RESULT);
+
   const ctx = createExecutionContext();
 
   const res = await worker.fetch(
@@ -74,6 +125,31 @@ test('GET `/dist/unknown-directory/` returns 404', async () => {
   expect(await res.text()).toStrictEqual('Directory not found');
 });
 
+test('GET `/dist/missing-in-kv-but-in-r2/` returns 200', async () => {
+  fetchMock
+    .get(s3Url.origin)
+    .intercept({
+      path: /.*/,
+      query: {
+        prefix: 'nodejs/release/missing-in-kv-but-in-r2/',
+      },
+    })
+    .reply(200, S3_DIRECTORY_RESULT);
+
+  const ctx = createExecutionContext();
+
+  const res = await worker.fetch(
+    new Request(`https://localhost/dist/missing-in-kv-but-in-r2/`),
+    mockedEnv,
+    ctx
+  );
+
+  // Consume body promise
+  await res.text();
+
+  expect(res.status).toBe(200);
+});
+
 test('GET `/dist` redirects to `/dist/`', async () => {
   const ctx = createExecutionContext();
 

scripts/constants.mjs

@@ -61,11 +61,4 @@ export const STATIC_FILE_SYMLINKS_PATH = join(
  * Paths in the R2 bucket that we should always be updating whenever a new
  * version releases.
  */
-export const ALWAYS_UPDATED_PATHS = [
-  RELEASE_DIR,
-  DOCS_DIR,
-  NIGHTLY_RELEASES_DIR,
-  RC_RELEASES_DIR,
-  TEST_RELEASES_DIR,
-  V8_CANARY_RELEASES_DIR,
-];
+export const ALWAYS_UPDATED_PATHS = [RELEASE_DIR, DOCS_DIR];

scripts/update-directory-cache.mjs → scripts/handle-mainline-release.mjs

@@ -1,11 +1,10 @@
 #!/usr/bin/env node
 
 /**
- * Update the most commonly updated paths in the directory cache for a Node.js
- * release.
+ * Updates the cached directory listings for paths that have symlinks for the
+ * mainline release bucket.
  *
- * Usage: ./update-directory-cache.mjs [version]
- * Example: ./update-directory-cache.mjs v24.0.0
+ * Usage: ./handle-mainline-release.mjs
  *
  * If no version is provided, this just updates what's in {@link ALWAYS_UPDATED_PATHS}
  */
@@ -22,8 +21,6 @@ import { getLatestVersionMapping } from './utils/getLatestVersionMapping.mjs';
 import { createCloudflareClient, writeKeysToKv } from './utils/kv.mjs';
 import { createS3Client, listR2DirectoryRecursive } from './utils/r2.mjs';
 
-const VERSION = process.argv[2]?.toLowerCase();
-
 // Ensure all necessary environment variables are set
 for (const envVar of [
   'CLOUDFLARE_API_TOKEN',
@@ -67,26 +64,6 @@ await Promise.all(
   )
 );
 
-if (VERSION !== undefined && VERSION !== '') {
-  if (!VERSION.startsWith('v')) {
-    throw new TypeError(
-      'provided version not in correct format, expected vX.X.X'
-    );
-  }
-
-  console.log(`Listing version ${VERSION}`);
-
-  const versionDirectory = await listR2DirectoryRecursive(
-    s3Client,
-    PROD_BUCKET,
-    `${RELEASE_DIR}${VERSION}/`
-  );
-
-  for (const [k, v] of versionDirectory.entries()) {
-    directories.set(k, v);
-  }
-}
-
 const latestVersions = await getLatestVersionMapping(
   s3Client,
   directories.get(RELEASE_DIR),
@@ -95,6 +72,8 @@ const latestVersions = await getLatestVersionMapping(
 
 await addSymlinksToDirectoryCache(s3Client, directories, latestVersions);
 
-// Update keys in the directory cache
+// Update keys in the directory cache. These intentionally do not have an
+// expiration set since they were made with info that the worker doesn't have
+// when deployed.
 console.log(`Writing ${directories.size} keys to directory cache...`);
 await writeKeysToKv(cfClient, DIRECTORY_CACHE_NAMESPACE_ID, directories);

scripts/utils/addSymlinksToDirectoryCache.mjs

@@ -124,10 +124,10 @@ export async function addStaticFileSymlinksToCache(
    */
   const fileSymlinks = JSON.parse(fileSymlinksFile);
 
-  // This file is an exception for two reasons:
-  //  1. It's dynamic and points to the latest Node version
-  //  2. It's handled specially and is technically in the root directory,
-  //     which we should never be listing thus don't need to add it to the
+  // These files are an exception for two reasons:
+  //  1. They're dynamic and point to the latest Node version
+  //  2. They're handled specially and are technically in the root directory,
+  //     which we should never be listing thus don't need to add them to the
   //     directory cache.
   // So, let's remove them then add them back once we're done handling the rest
   // of the static file symlinks.

src/constants/cache.ts

@@ -4,3 +4,8 @@ export const CACHE_HEADERS = {
   mutable: 'public, max-age=3600, s-maxage=14400',
   failure: 'private, no-cache, no-store, max-age=0, must-revalidate',
 };
+
+/**
+ * Ttl for elements in the KV cache in seconds. This is completely arbitrary
+ */
+export const KV_CACHE_TTL = 60 * 60 * 6;