fix(storage): don't watch local files + improved queue busy logic

Akryum · Sep 4, 2024 · 68bf368 · 68bf368
1 parent c3601df
commit 68bf368
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 25 deletions.
diff --git a/packages/core/src/storage/storage.ts b/packages/core/src/storage/storage.ts
@@ -69,6 +69,7 @@ export async function useStorage<TData extends { id: string }>(mq: MoquerieInsta
     : getRepositoryDbFolder(mq)
   const folder = path.join(baseFolder, options.path)
   const manifestFile = path.join(folder, 'manifest.json')
+  const shouldWatch = options.location !== 'local' && !mq.data.skipWrites && mq.data.watching
 
   await ensureDir(folder)
 
@@ -152,29 +153,6 @@ export async function useStorage<TData extends { id: string }>(mq: MoquerieInsta
     return item
   }
 
-  async function load() {
-    data.length = 0
-
-    if (!options.lazyLoading) {
-      const promises = []
-      for (const id in manifest.files) {
-        promises.push((async () => {
-          const file = manifest.files[id]
-          try {
-            const item = await readFile(id, file)
-            data.push(item)
-          }
-          catch (e) {
-            console.error(e)
-          }
-        })())
-      }
-      await Promise.all(promises)
-    }
-  }
-
-  await load()
-
   // Write
 
   const writeQueue = useQueue({
@@ -318,9 +296,9 @@ export async function useStorage<TData extends { id: string }>(mq: MoquerieInsta
 
   let refreshInterval: NodeJS.Timeout
 
-  if (!mq.data.skipWrites && mq.data.watching) {
+  if (shouldWatch) {
     refreshInterval = setInterval(async () => {
-      if (!writeQueue.size) {
+      if (!writeQueue.busy) {
         manifest = await readManifest()
         await load()
       }
@@ -410,6 +388,37 @@ export async function useStorage<TData extends { id: string }>(mq: MoquerieInsta
     }
   }
 
+  // Load
+
+  async function load() {
+    const newData: TData[] = []
+
+    if (!options.lazyLoading) {
+      const promises = []
+      for (const id in manifest.files) {
+        promises.push((async () => {
+          const file = manifest.files[id]
+          try {
+            const item = await readFile(id, file)
+            newData.push(item)
+          }
+          catch (e) {
+            console.error(e)
+          }
+        })())
+      }
+      await Promise.all(promises)
+    }
+
+    // Only update data if no write is pending
+    if (!writeQueue.busy) {
+      data.length = 0
+      data.push(...newData)
+    }
+  }
+
+  await load()
+
   // Cleanup
 
   function destroy() {

diff --git a/packages/core/src/util/queue.ts b/packages/core/src/util/queue.ts
@@ -7,6 +7,7 @@ export interface UseQueueOptions {
 
 export function useQueue(options: UseQueueOptions) {
   const queued = new Map<string, (() => Promise<void>) | null>()
+  const processing = new Set<string>()
 
   function queue(key: string, callback: () => Promise<void>) {
     const alreadyQueued = queued.has(key)
@@ -31,8 +32,10 @@ export function useQueue(options: UseQueueOptions) {
   function run(key: string) {
     const callback = queued.get(key)
     if (callback) {
+      processing.add(key)
       queued.delete(key)
       callback().finally(() => {
+        processing.delete(key)
         if (queued.has(key)) {
           next(key)
         }
@@ -45,5 +48,8 @@ export function useQueue(options: UseQueueOptions) {
     get size() {
       return queued.size
     },
+    get busy() {
+      return queued.size > 0 || processing.size > 0
+    },
   }
 }