Skip to content

Commit

Permalink
add cloudflare caching worker
Browse files Browse the repository at this point in the history
proxy all files instead of using signed bucket urls
  • Loading branch information
mhuebert committed Nov 7, 2024
1 parent e3d7531 commit 5b1f8a5
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 156 deletions.
7 changes: 3 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
"dependencies": {
"@google-cloud/storage": "^7.11.1",
"express": "^4.19.2",
"memoizee": "^0.4.17",
"stream-mime-type": "^2.0.0",
"wrangler": "^3.85.0"
"memoizee": "^0.4.17"
},
"scripts": {
"start": "node src/start.mjs",
Expand All @@ -23,7 +21,8 @@
"create-cloudflare": "^2.31.2",
"mocha": "^10.4.0",
"nodemon": "^3.1.0",
"supertest": "^7.0.0"
"supertest": "^7.0.0",
"wrangler": "^3.85.0"
},
"type": "module"
}
67 changes: 18 additions & 49 deletions private-website-cache/src/worker.js
Original file line number Diff line number Diff line change
@@ -1,56 +1,25 @@
export default {
async fetch(request, env, ctx) {
const url = new URL(request.url);
const extension = url.pathname.split('.').pop();

// Forward the request to the origin
const response = await fetch(request);
const modifiedResponse = new Response(response.body, response);

// Clear any existing cache headers
modifiedResponse.headers.delete('Pragma');
modifiedResponse.headers.delete('Cache-Control');
modifiedResponse.headers.delete('Expires');

// Check if this is a redirect to a signed URL
if (response.status === 302 && response.headers.get('Location')?.includes('storage.googleapis.com')) {
// Signed URL redirects: private, 50 minutes
const maxAge = 50 * 60; // 50 minutes in seconds
modifiedResponse.headers.set('Cache-Control', `private, max-age=${maxAge}`);
const expiresDate = new Date(Date.now() + maxAge * 1000);
modifiedResponse.headers.set('Expires', expiresDate.toUTCString());
} else if (extension) {
// Files with extensions
switch (extension.toLowerCase()) {
case 'html':
// HTML files: private, 60 seconds
modifiedResponse.headers.set('Cache-Control', 'private, max-age=60');
break;

case 'css':
case 'js':
case 'map':
case 'wasm':
case 'data':
case 'bin': // Binary files often used with WebGL
case 'gltf': // 3D models
case 'glb': // Binary 3D models
case 'obj': // 3D models
case 'mtl': // Material files for 3D models
case 'json': // Often used for model/scene data
// Static assets: private, 10 minutes
modifiedResponse.headers.set('Cache-Control', 'private, max-age=600');
break;

default:
// Other static assets that aren't redirects: private, short cache
modifiedResponse.headers.set('Cache-Control', 'private, max-age=60');
// Check for custom cache policy
const cachePolicyHeader = response.headers.get('X-Cache-Policy');
if (cachePolicyHeader) {
try {
const policy = JSON.parse(cachePolicyHeader);
const modifiedResponse = new Response(response.body, response);
modifiedResponse.headers.delete('Pragma');
modifiedResponse.headers.delete('Cache-Control');
modifiedResponse.headers.delete('Expires');
modifiedResponse.headers.set('Cache-Control', `${policy.visibility}, max-age=${policy.maxAge}`);
const expiresDate = new Date(Date.now() + policy.maxAge * 1000);
modifiedResponse.headers.set('Expires', expiresDate.toUTCString());
return modifiedResponse;
} catch (e) {
console.error('Failed to parse cache policy:', e);
}
} else {
// Paths without extensions (HTML fallbacks): private, 60 seconds
modifiedResponse.headers.set('Cache-Control', 'private, max-age=60');
}
return modifiedResponse;

return response;
}
};
};
58 changes: 27 additions & 31 deletions src/server.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import { Storage } from '@google-cloud/storage';
import express from 'express';
import memoizee from 'memoizee';
import path from 'path';
import { getMimeType } from 'stream-mime-type';
import * as assert from 'assert'

const { BUCKET_NAME } = process.env;
Expand Down Expand Up @@ -49,39 +48,39 @@ const handleResponseError = (res, error) => {
}
}

const pipeFile = async (res, path) => {
const CACHE_POLICIES = {
HTML: { maxAge: 60, visibility: 'private' },
STATIC: { maxAge: 600, visibility: 'public' },
SIGNED_URL: { maxAge: 3000, visibility: 'public' },
};

const setCacheHeaders = (res, policy) => {
res.setHeader('X-Cache-Policy', JSON.stringify(policy));
res.setHeader('Cache-Control', `${policy.visibility}, max-age=${policy.maxAge}`);
res.setHeader('Expires', new Date(Date.now() + policy.maxAge * 1000).toUTCString());
};

const serveFile = async (res, path) => {
try {
const bucketStream = default_bucket.file(path).createReadStream()
bucketStream.on('error', (error) => handleResponseError(res, error))
const { mime, stream } = await getMimeType(bucketStream, { filename: path });
res.setHeader('Content-Type', mime);
res.setHeader('Cache-Control', `private, max-age=${HTML_MAX_AGE}`);
const file = default_bucket.file(path);
const [metadata] = await file.getMetadata();

setCacheHeaders(res, CACHE_POLICIES.STATIC);
res.setHeader('Content-Type', metadata.contentType);
res.setHeader('Content-Length', metadata.size);

const stream = file.createReadStream();
stream.on('error', (err) => handleResponseError(res, err));
return stream.pipe(res);
} catch (err) {
handleResponseError(res, err)
handleResponseError(res, err);
}
};

const redirectFile = async (res, path) => {
// Redirects static asset requests to signed bucket URLs instead of piping them through
// this server. Signed URLs are valid for one hour, and memoized.

// Note: IAP (Identity Aware Proxy) adds cache-busting headers to all requests to prevent
// caching of private content. These headers *should* only control the redirect itself,
// but they cause Safari to refuse to cache the destination as well.

const [signedUrl, expires] = await generateSignedUrl(BUCKET_NAME, path);
const maxAge = (expires - Date.now()) / 1000; // Calculate max-age in seconds
res.setHeader('Expires', new Date(expires).toUTCString());
res.setHeader('Cache-Control', `private, max-age=${maxAge}`);
res.redirect(302, signedUrl);
};

const serveHtml = async (res, path) => {
const htmlFile = default_bucket.file(path);
res.setHeader('Content-Type', 'text/html');
res.setHeader('Cache-Control', `private, max-age=${HTML_MAX_AGE}`);
setCacheHeaders(res, CACHE_POLICIES.HTML);
return new Promise((resolve, reject) => {
let rejectLogged = (err) => {
reject(err)
Expand Down Expand Up @@ -162,22 +161,19 @@ const handleRequest = async (parentDomain, subDomain, filePath, req, res) => {
return;
}

const fileExtension = getExtension(filePath);
// Paths with non-html file extensions redirect to the bucket
const fileExtension = getExtension(filePath);
try {
if (fileExtension) {
if (fileExtension === 'html') {
await serveHtml(res, path.join(parentDomain, subDomain, filePath));
} else if (fileExtension === 'css' || fileExtension === 'js' || fileExtension == "map" || req.headers['sec-fetch-dest'] == 'script') {
await pipeFile(res, path.join(parentDomain, subDomain, filePath));
} else {
await redirectFile(res, path.join(parentDomain, subDomain, filePath));
await serveFile(res, path.join(parentDomain, subDomain, filePath));
}
} else {
await serveHtmlWithFallbacks(res, parentDomain, subDomain, paths(filePath));
}
}
} catch (error) {
handleResponseError(res, error)
handleResponseError(res, error);
}
};

Expand Down
74 changes: 2 additions & 72 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -234,11 +234,6 @@
"@jridgewell/resolve-uri" "^3.0.3"
"@jridgewell/sourcemap-codec" "^1.4.10"

"@tokenizer/token@^0.3.0":
version "0.3.0"
resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.3.0.tgz#fe98a93fe789247e998c75e74e9c7c63217aa276"
integrity sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==

"@tootallnate/once@2":
version "2.0.0"
resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf"
Expand Down Expand Up @@ -942,15 +937,6 @@ fast-xml-parser@^4.3.0:
dependencies:
strnum "^1.0.5"

file-type@^18.2.0:
version "18.7.0"
resolved "https://registry.yarnpkg.com/file-type/-/file-type-18.7.0.tgz#cddb16f184d6b94106cfc4bb56978726b25cb2a2"
integrity sha512-ihHtXRzXEziMrQ56VSgU7wkxh55iNchFkosu7Y9/S+tXHdKyrGjVK0ujbqNnsxzea+78MaLhN6PGmfYSAv1ACw==
dependencies:
readable-web-to-node-stream "^3.0.2"
strtok3 "^7.0.0"
token-types "^5.0.1"

fill-range@^7.1.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292"
Expand Down Expand Up @@ -1226,11 +1212,6 @@ iconv-lite@0.4.24:
dependencies:
safer-buffer ">= 2.1.2 < 3"

ieee754@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==

ignore-by-default@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/ignore-by-default/-/ignore-by-default-1.0.1.tgz#48ca6d72f6c6a3af00a9ad4ae6876be3889e2b09"
Expand Down Expand Up @@ -1416,7 +1397,7 @@ mime-db@1.52.0:
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==

mime-types@^2.1.12, mime-types@^2.1.35, mime-types@~2.1.24, mime-types@~2.1.34:
mime-types@^2.1.12, mime-types@~2.1.24, mime-types@~2.1.34:
version "2.1.35"
resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==
Expand Down Expand Up @@ -1644,11 +1625,6 @@ pathval@^2.0.0:
resolved "https://registry.yarnpkg.com/pathval/-/pathval-2.0.0.tgz#7e2550b422601d4f6b8e26f1301bc8f15a741a25"
integrity sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==

peek-readable@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/peek-readable/-/peek-readable-5.0.0.tgz#7ead2aff25dc40458c60347ea76cfdfd63efdfec"
integrity sha512-YtCKvLUOvwtMGmrniQPdO7MwPjgkFBtFIrmfSbYmYuq3tKDV/mcfAhBth1+C3ru7uXIZasc/pHnb+YDYNkkj4A==

picomatch@^2.0.4, picomatch@^2.2.1:
version "2.3.1"
resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
Expand Down Expand Up @@ -1708,7 +1684,7 @@ raw-body@2.5.2:
iconv-lite "0.4.24"
unpipe "1.0.0"

readable-stream@3, readable-stream@^3.1.1, readable-stream@^3.6.0:
readable-stream@^3.1.1:
version "3.6.2"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967"
integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==
Expand All @@ -1717,13 +1693,6 @@ readable-stream@3, readable-stream@^3.1.1, readable-stream@^3.6.0:
string_decoder "^1.1.1"
util-deprecate "^1.0.1"

readable-web-to-node-stream@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz#5d52bb5df7b54861fd48d015e93a2cb87b3ee0bb"
integrity sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==
dependencies:
readable-stream "^3.6.0"

readdirp@~3.6.0:
version "3.6.0"
resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7"
Expand Down Expand Up @@ -1915,22 +1884,6 @@ stream-events@^1.0.5:
dependencies:
stubs "^3.0.0"

stream-head@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/stream-head/-/stream-head-3.0.0.tgz#cf69c14f3f6d8c63b1475a0e3ccc0ee58ddd2c1c"
integrity sha512-EfcHQpe+HxwAY46J+o+LeQG8gL6FfxBBfNEGzPWzXYEiL2dRS1dtFJ2F38JLcrSKz1tIFA3HkST4SkTPA7+jgw==
dependencies:
through2 "4.0.2"

stream-mime-type@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/stream-mime-type/-/stream-mime-type-2.0.0.tgz#584aeeea2f45bd1a732d8bd248f8bcfba1130bad"
integrity sha512-JKka2v3YRBr0P2nWeJIWTBtMhYO1CHQp/TecKuSdz6+33Tm5Z5xH2raLalkVrRYloHa3FZKAe7DCVAQeRPYUGQ==
dependencies:
file-type "^18.2.0"
mime-types "^2.1.35"
stream-head "^3.0.0"

stream-shift@^1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.3.tgz#85b8fab4d71010fc3ba8772e8046cc49b8a3864b"
Expand Down Expand Up @@ -1969,14 +1922,6 @@ strnum@^1.0.5:
resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db"
integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==

strtok3@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-7.0.0.tgz#868c428b4ade64a8fd8fee7364256001c1a4cbe5"
integrity sha512-pQ+V+nYQdC5H3Q7qBZAz/MO6lwGhoC2gOAjuouGf/VO0m7vQRh8QNMl2Uf6SwAtzZ9bOw3UIeBukEGNJl5dtXQ==
dependencies:
"@tokenizer/token" "^0.3.0"
peek-readable "^5.0.0"

stubs@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/stubs/-/stubs-3.0.0.tgz#e8d2ba1fa9c90570303c030b6900f7d5f89abe5b"
Expand Down Expand Up @@ -2042,13 +1987,6 @@ teeny-request@^9.0.0:
stream-events "^1.0.5"
uuid "^9.0.0"

through2@4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764"
integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==
dependencies:
readable-stream "3"

timers-ext@^0.1.7:
version "0.1.7"
resolved "https://registry.yarnpkg.com/timers-ext/-/timers-ext-0.1.7.tgz#6f57ad8578e07a3fb9f91d9387d65647555e25c6"
Expand All @@ -2069,14 +2007,6 @@ toidentifier@1.0.1:
resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35"
integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==

token-types@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/token-types/-/token-types-5.0.1.tgz#aa9d9e6b23c420a675e55413b180635b86a093b4"
integrity sha512-Y2fmSnZjQdDb9W4w4r1tswlMHylzWIeOKpx0aZH9BgGtACHhrk3OkT52AzwcuqTRBZtvvnTjDBh8eynMulu8Vg==
dependencies:
"@tokenizer/token" "^0.3.0"
ieee754 "^1.2.1"

touch@^3.1.0:
version "3.1.1"
resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.1.tgz#097a23d7b161476435e5c1344a95c0f75b4a5694"
Expand Down

0 comments on commit 5b1f8a5

Please sign in to comment.