11import { getDeployStore , GetWithMetadataOptions , Store } from '@netlify/blobs'
2+ import { LRUCache } from 'lru-cache'
23
3- import type { BlobType } from '../shared/cache-types.cjs'
4+ import { type BlobType , estimateBlobSize } from '../shared/cache-types.cjs'
45
6+ import { getRequestContext } from './handlers/request-context.cjs'
57import { getTracer } from './handlers/tracer.cjs'
68
79const FETCH_BEFORE_NEXT_PATCHED_IT = Symbol . for ( 'nf-not-patched-fetch' )
@@ -70,6 +72,53 @@ const encodeBlobKey = async (key: string) => {
7072 return await encodeBlobKeyImpl ( key )
7173}
7274
75+ // lru-cache types don't like using `null` for values, so we use a symbol to represent it and do conversion
76+ // so it doesn't leak outside
77+ const NullValue = Symbol . for ( 'null-value' )
78+ const inMemoryLRUCache = new LRUCache <
79+ string ,
80+ BlobType | typeof NullValue | Promise < BlobType | null >
81+ > ( {
82+ max : 1000 , // recommended to set a limit for perf, unlikely to hit it in practice, as eviction will be primarily size based
83+ maxSize : 50 * 1024 * 1024 , // 50MB
84+ sizeCalculation : ( valueToStore ) => {
85+ return estimateBlobSize ( valueToStore === NullValue ? null : valueToStore )
86+ } ,
87+ } )
88+
89+ interface RequestSpecificInMemoryCache {
90+ get ( key : string ) : BlobType | null | Promise < BlobType | null > | undefined
91+ set ( key : string , value : BlobType | null | Promise < BlobType | null > ) : void
92+ }
93+
94+ const getRequestSpecificInMemoryCache = ( ) : RequestSpecificInMemoryCache => {
95+ const requestContext = getRequestContext ( )
96+ if ( ! requestContext ) {
97+ // Fallback to a no-op store if we can't find request context
98+ return {
99+ get ( ) : undefined {
100+ // no-op
101+ } ,
102+ set ( ) {
103+ // no-op
104+ } ,
105+ }
106+ }
107+
108+ return {
109+ get ( key ) {
110+ const inMemoryValue = inMemoryLRUCache . get ( `${ requestContext . requestID } ${ key } )
111+ if ( inMemoryValue === NullValue ) {
112+ return null
113+ }
114+ return inMemoryValue
115+ } ,
116+ set ( key , value ) {
117+ inMemoryLRUCache . set ( `${ requestContext . requestID } ${ key } , value ?? NullValue )
118+ } ,
119+ }
120+ }
121+
73122export const getMemoizedKeyValueStoreBackedByRegionalBlobStore = (
74123 args : GetWithMetadataOptions = { } ,
75124) => {
@@ -78,18 +127,32 @@ export const getMemoizedKeyValueStoreBackedByRegionalBlobStore = (
78127
79128 return {
80129 async get < T extends BlobType > ( key : string , otelSpanTitle : string ) : Promise < T | null > {
81- const blobKey = await encodeBlobKey ( key )
130+ const inMemoryCache = getRequestSpecificInMemoryCache ( )
82131
83- return tracer . withActiveSpan ( otelSpanTitle , async ( span ) => {
132+ const memoizedValue = inMemoryCache . get ( key )
133+ if ( typeof memoizedValue !== 'undefined' ) {
134+ return memoizedValue as T | null | Promise < T | null >
135+ }
136+
137+ const blobKey = await encodeBlobKey ( key )
138+ const getPromise = tracer . withActiveSpan ( otelSpanTitle , async ( span ) => {
84139 span . setAttributes ( { key, blobKey } )
85140 const blob = ( await store . get ( blobKey , { type : 'json' } ) ) as T | null
141+ inMemoryCache . set ( key , blob )
142+ console . log ( 'after set value size 1' , inMemoryLRUCache . calculatedSize )
86143 span . addEvent ( blob ? 'Hit' : 'Miss' )
87144 return blob
88145 } )
146+ inMemoryCache . set ( key , getPromise )
147+ return getPromise
89148 } ,
90149 async set ( key : string , value : BlobType , otelSpanTitle : string ) {
91- const blobKey = await encodeBlobKey ( key )
150+ const inMemoryCache = getRequestSpecificInMemoryCache ( )
92151
152+ inMemoryCache . set ( key , value )
153+ console . log ( 'after set value size 2' , inMemoryLRUCache . calculatedSize )
154+
155+ const blobKey = await encodeBlobKey ( key )
93156 return tracer . withActiveSpan ( otelSpanTitle , async ( span ) => {
94157 span . setAttributes ( { key, blobKey } )
95158 return await store . setJSON ( blobKey , value )
0 commit comments