Skip to content

Commit 5bc519f

Browse files
piehpieh
committed
perf: use conditional blob gets if same blob was fetched before for previous requests
1 parent c85acf8 commit 5bc519f

File tree

4 files changed

+312
-88
lines changed

4 files changed

+312
-88
lines changed

‎.eslintrc.cjs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ module.exports = {
3737
},
3838
],
3939
'unicorn/filename-case': ['error', { case: 'kebabCase' }],
40+
'unicorn/no-nested-ternary': 'off',
4041
'unicorn/numeric-separators-style': 'off',
4142
},
4243
overrides: [

‎src/run/storage/request-scoped-in-memory-cache.cts‎

Lines changed: 115 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,34 @@ import { recordWarning } from '../handlers/tracer.cjs'
99
// lru-cache types don't like using `null` for values, so we use a symbol to represent it and do conversion
1010
// so it doesn't leak outside
1111
const NullValue = Symbol.for('null-value')
12-
type BlobLRUCache = LRUCache<string, BlobType | typeof NullValue | Promise<BlobType | null>>
12+
type DataWithEtag = { data: BlobType; etag: string }
13+
14+
const isDataWithEtag = (value: unknown): value is DataWithEtag => {
15+
return typeof value === 'object' && value !== null && 'data' in value && 'etag' in value
16+
}
17+
18+
type BlobLRUCache = LRUCache<
19+
string,
20+
BlobType | typeof NullValue | Promise<BlobType | null> | DataWithEtag
21+
>
1322

1423
const IN_MEMORY_CACHE_MAX_SIZE = Symbol.for('nf-in-memory-cache-max-size')
1524
const IN_MEMORY_LRU_CACHE = Symbol.for('nf-in-memory-lru-cache')
1625
const extendedGlobalThis = globalThis as typeof globalThis & {
1726
[IN_MEMORY_CACHE_MAX_SIZE]?: number
18-
[IN_MEMORY_LRU_CACHE]?: BlobLRUCache | null
27+
[IN_MEMORY_LRU_CACHE]?: {
28+
/**
29+
* entries are scoped to request IDs
30+
*/
31+
perRequest: BlobLRUCache
32+
/**
33+
* global cache shared between requests, does not allow immediate re-use, but is used for
34+
* conditional blob gets with etags and given blob key is first tried in given request.
35+
* Map values are weak references to avoid this map strongly referencing blobs and allowing
36+
* GC based on per request LRU cache evictions alone.
37+
*/
38+
global: Map<string, WeakRef<DataWithEtag>>
39+
} | null
1940
}
2041

2142
const DEFAULT_FALLBACK_MAX_SIZE = 50 * 1024 * 1024 // 50MB, same as default Next.js config
@@ -31,40 +52,46 @@ const isPositiveNumber = (value: unknown): value is PositiveNumber => {
3152
}
3253

3354
const BASE_BLOB_SIZE = 25 as PositiveNumber
55+
const BASE_BLOB_WITH_ETAG_SIZE = (BASE_BLOB_SIZE + 34) as PositiveNumber
3456

3557
const estimateBlobKnownTypeSize = (
36-
valueToStore: BlobType | null | Promise<unknown>,
58+
valueToStore: BlobType | null | Promise<unknown> | DataWithEtag,
3759
): number | undefined => {
3860
// very approximate size calculation to avoid expensive exact size calculation
3961
// inspired by https://github.com/vercel/next.js/blob/ed10f7ed0246fcc763194197eb9beebcbd063162/packages/next/src/server/lib/incremental-cache/file-system-cache.ts#L60-L79
40-
if (valueToStore === null || isPromise(valueToStore) || isTagManifest(valueToStore)) {
62+
if (valueToStore === null || isPromise(valueToStore)) {
4163
return BASE_BLOB_SIZE
4264
}
43-
if (isHtmlBlob(valueToStore)) {
44-
return BASE_BLOB_SIZE + valueToStore.html.length
65+
66+
const { data, baseSize } = isDataWithEtag(valueToStore)
67+
? { data: valueToStore.data, baseSize: BASE_BLOB_WITH_ETAG_SIZE }
68+
: { data: valueToStore, baseSize: BASE_BLOB_SIZE }
69+
70+
if (isTagManifest(data)) {
71+
return baseSize
72+
}
73+
74+
if (isHtmlBlob(data)) {
75+
return baseSize + data.html.length
4576
}
4677

47-
if (valueToStore.value?.kind === 'FETCH') {
48-
return BASE_BLOB_SIZE + valueToStore.value.data.body.length
78+
if (data.value?.kind === 'FETCH') {
79+
return baseSize + data.value.data.body.length
4980
}
50-
if (valueToStore.value?.kind === 'APP_PAGE') {
51-
return (
52-
BASE_BLOB_SIZE + valueToStore.value.html.length + (valueToStore.value.rscData?.length ?? 0)
53-
)
81+
if (data.value?.kind === 'APP_PAGE') {
82+
return baseSize + data.value.html.length + (data.value.rscData?.length ?? 0)
5483
}
55-
if (valueToStore.value?.kind === 'PAGE' || valueToStore.value?.kind === 'PAGES') {
56-
return (
57-
BASE_BLOB_SIZE +
58-
valueToStore.value.html.length +
59-
JSON.stringify(valueToStore.value.pageData).length
60-
)
84+
if (data.value?.kind === 'PAGE' || data.value?.kind === 'PAGES') {
85+
return baseSize + data.value.html.length + JSON.stringify(data.value.pageData).length
6186
}
62-
if (valueToStore.value?.kind === 'ROUTE' || valueToStore.value?.kind === 'APP_ROUTE') {
63-
return BASE_BLOB_SIZE + valueToStore.value.body.length
87+
if (data.value?.kind === 'ROUTE' || data.value?.kind === 'APP_ROUTE') {
88+
return baseSize + data.value.body.length
6489
}
6590
}
6691

67-
const estimateBlobSize = (valueToStore: BlobType | null | Promise<unknown>): PositiveNumber => {
92+
const estimateBlobSize = (
93+
valueToStore: BlobType | null | Promise<unknown> | DataWithEtag,
94+
): PositiveNumber => {
6895
let estimatedKnownTypeSize: number | undefined
6996
let estimateBlobKnownTypeSizeError: unknown
7097
try {
@@ -98,23 +125,45 @@ function getInMemoryLRUCache() {
98125
? extendedGlobalThis[IN_MEMORY_CACHE_MAX_SIZE]
99126
: DEFAULT_FALLBACK_MAX_SIZE
100127

101-
extendedGlobalThis[IN_MEMORY_LRU_CACHE] =
102-
maxSize === 0
103-
? null // if user sets 0 in their config, we should honor that and not use in-memory cache
104-
: new LRUCache<string, BlobType | typeof NullValue | Promise<BlobType | null>>({
105-
max: 1000,
106-
maxSize,
107-
sizeCalculation: (valueToStore) => {
108-
return estimateBlobSize(valueToStore === NullValue ? null : valueToStore)
109-
},
110-
})
128+
if (maxSize === 0) {
129+
extendedGlobalThis[IN_MEMORY_LRU_CACHE] = null
130+
} else {
131+
const global = new Map<string, WeakRef<DataWithEtag>>()
132+
133+
const perRequest = new LRUCache<
134+
string,
135+
BlobType | typeof NullValue | Promise<BlobType | null> | DataWithEtag
136+
>({
137+
max: 1000,
138+
maxSize,
139+
sizeCalculation: (valueToStore) => {
140+
return estimateBlobSize(valueToStore === NullValue ? null : valueToStore)
141+
},
142+
})
143+
144+
extendedGlobalThis[IN_MEMORY_LRU_CACHE] = {
145+
perRequest,
146+
global,
147+
}
148+
}
111149
}
112150
return extendedGlobalThis[IN_MEMORY_LRU_CACHE]
113151
}
114152

153+
export function clearInMemoryLRUCacheForTesting() {
154+
extendedGlobalThis[IN_MEMORY_LRU_CACHE] = undefined
155+
}
156+
115157
interface RequestScopedInMemoryCache {
116-
get(key: string): BlobType | null | Promise<BlobType | null> | undefined
117-
set(key: string, value: BlobType | null | Promise<BlobType | null>): void
158+
get(key: string):
159+
| { conditional: false; currentRequestValue: BlobType | null | Promise<BlobType | null> }
160+
| {
161+
conditional: true
162+
globalValue: BlobType
163+
etag: string
164+
}
165+
| undefined
166+
set(key: string, value: BlobType | null | Promise<BlobType | null> | DataWithEtag): void
118167
}
119168

120169
export const getRequestScopedInMemoryCache = (): RequestScopedInMemoryCache => {
@@ -125,8 +174,35 @@ export const getRequestScopedInMemoryCache = (): RequestScopedInMemoryCache => {
125174
get(key) {
126175
if (!requestContext) return
127176
try {
128-
const value = inMemoryLRUCache?.get(`${requestContext.requestID}:${key}`)
129-
return value === NullValue ? null : value
177+
const currentRequestValue = inMemoryLRUCache?.perRequest.get(
178+
`${requestContext.requestID}:${key}`,
179+
)
180+
if (currentRequestValue) {
181+
return {
182+
conditional: false,
183+
currentRequestValue:
184+
currentRequestValue === NullValue
185+
? null
186+
: isDataWithEtag(currentRequestValue)
187+
? currentRequestValue.data
188+
: currentRequestValue,
189+
}
190+
}
191+
192+
const globalEntry = inMemoryLRUCache?.global.get(key)
193+
if (globalEntry) {
194+
const derefencedGlobalEntry = globalEntry.deref()
195+
if (derefencedGlobalEntry) {
196+
return {
197+
conditional: true,
198+
globalValue: derefencedGlobalEntry.data,
199+
etag: derefencedGlobalEntry.etag,
200+
}
201+
}
202+
203+
// value has been GC'ed so we can cleanup entry from the map as it no longer points to existing value
204+
inMemoryLRUCache?.global.delete(key)
205+
}
130206
} catch (error) {
131207
// using in-memory store is perf optimization not requirement
132208
// trying to use optimization should NOT cause crashes
@@ -137,7 +213,10 @@ export const getRequestScopedInMemoryCache = (): RequestScopedInMemoryCache => {
137213
set(key, value) {
138214
if (!requestContext) return
139215
try {
140-
inMemoryLRUCache?.set(`${requestContext?.requestID}:${key}`, value ?? NullValue)
216+
if (isDataWithEtag(value)) {
217+
inMemoryLRUCache?.global.set(key, new WeakRef(value))
218+
}
219+
inMemoryLRUCache?.perRequest.set(`${requestContext.requestID}:${key}`, value ?? NullValue)
141220
} catch (error) {
142221
// using in-memory store is perf optimization not requirement
143222
// trying to use optimization should NOT cause crashes

‎src/run/storage/storage.cts‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,43 @@ export const getMemoizedKeyValueStoreBackedByRegionalBlobStore = (
2525
const inMemoryCache = getRequestScopedInMemoryCache()
2626

2727
const memoizedValue = inMemoryCache.get(key)
28-
if (typeof memoizedValue !== 'undefined') {
29-
return memoizedValue as T | null | Promise<T | null>
28+
if (
29+
memoizedValue?.conditional === false &&
30+
typeof memoizedValue?.currentRequestValue !== 'undefined'
31+
) {
32+
return memoizedValue.currentRequestValue as T | null | Promise<T | null>
3033
}
3134

3235
const blobKey = await encodeBlobKey(key)
3336
const getPromise = withActiveSpan(tracer, otelSpanTitle, async (span) => {
34-
span?.setAttributes({ key, blobKey })
35-
const blob = (await store.get(blobKey, { type: 'json' })) as T | null
36-
inMemoryCache.set(key, blob)
37-
span?.addEvent(blob ? 'Hit' : 'Miss')
37+
const { etag: previousEtag, globalValue: previousBlob } = memoizedValue?.conditional
38+
? memoizedValue
39+
: {}
40+
41+
span?.setAttributes({ key, blobKey, previousEtag })
42+
43+
const result = await store.getWithMetadata(blobKey, {
44+
type: 'json',
45+
etag: previousEtag,
46+
})
47+
48+
const shouldReuseMemoizedBlob = result?.etag && previousEtag === result?.etag
49+
50+
const blob = (shouldReuseMemoizedBlob ? previousBlob : result?.data) as T | null
51+
52+
if (result?.etag && blob) {
53+
inMemoryCache.set(key, {
54+
data: blob,
55+
etag: result?.etag,
56+
})
57+
}
58+
59+
span?.setAttributes({
60+
etag: result?.etag,
61+
reusingPreviouslyFetchedBlob: shouldReuseMemoizedBlob,
62+
status: blob ? (shouldReuseMemoizedBlob ? 'Hit, no change' : 'Hit') : 'Miss',
63+
})
64+
3865
return blob
3966
})
4067
inMemoryCache.set(key, getPromise)
@@ -48,7 +75,14 @@ export const getMemoizedKeyValueStoreBackedByRegionalBlobStore = (
4875
const blobKey = await encodeBlobKey(key)
4976
return withActiveSpan(tracer, otelSpanTitle, async (span) => {
5077
span?.setAttributes({ key, blobKey })
51-
return await store.setJSON(blobKey, value)
78+
const writeResult = await store.setJSON(blobKey, value)
79+
if (writeResult?.etag) {
80+
inMemoryCache.set(key, {
81+
data: value,
82+
etag: writeResult.etag,
83+
})
84+
}
85+
return writeResult
5286
})
5387
},
5488
}

0 commit comments

Comments
 (0)