spx-gui: implement batched object URL processing and optimize lazy lo…

…ading

- Extended `util.makeObjectUrls` with batching to queue and merge object
  URL requests.
- Updated `cloud.universalUrlToWebUrl` to use batched processing.
- Refactored `cloud.getFiles` and file creation process to support lazy
  URL resolution, addressing potential URL expiry issues.

Fixes goplus#653

spx-gui/src/apis/util.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
+import { makeObjectUrls } from './util'
+import { client, type UniversalToWebUrlMap } from './common'
+
+vi.mock('./common', () => ({
+ client: {
+ post: vi.fn()
+ }
+}))
+
+describe('makeObjectUrls', () => {
+ beforeEach(() => {
+ vi.useFakeTimers()
+ })
+
+ afterEach(() => {
+ vi.useRealTimers()
+ vi.resetAllMocks()
+ })
+
+ it('should make object URLs correctly with immediate flag', async () => {
+ const mockResponse = {
+ objectUrls: {
+ 'kodo://bucket/key1': 'https://bucket.example.com/key1',
+ 'kodo://bucket/key2': 'https://bucket.example.com/key2'
+ } as UniversalToWebUrlMap
+ }
+ vi.mocked(client.post).mockResolvedValue(mockResponse)
+
+ const objects = ['kodo://bucket/key1', 'kodo://bucket/key2']
+ const objectUrls = await makeObjectUrls(objects, true)
+
+ expect(client.post).toHaveBeenCalledWith('/util/fileurls', { objects })
+ expect(objectUrls).toEqual(mockResponse.objectUrls)
+ })
+
+ it('should make multiple requests when immediate flag is true', async () => {
+ const mockResponse1 = {
+ objectUrls: {
+ 'kodo://bucket/key1': 'https://bucket.example.com/key1',
+ 'kodo://bucket/key2': 'https://bucket.example.com/key2'
+ } as UniversalToWebUrlMap
+ }
+ const mockResponse2 = {
+ objectUrls: {
+ 'kodo://bucket/key3': 'https://bucket.example.com/key3'
+ } as UniversalToWebUrlMap
+ }
+ vi.mocked(client.post).mockResolvedValueOnce(mockResponse1).mockResolvedValueOnce(mockResponse2)
+
+ const objects1 = ['kodo://bucket/key1', 'kodo://bucket/key2']
+ const objects2 = ['kodo://bucket/key3']
+ const [objectUrls1, objectUrls2] = await Promise.all([
+ makeObjectUrls(objects1, true),
+ makeObjectUrls(objects2, true)
+ ])
+
+ expect(client.post).toHaveBeenCalledTimes(2)
+ expect(client.post).toHaveBeenNthCalledWith(1, '/util/fileurls', { objects: objects1 })
+ expect(client.post).toHaveBeenNthCalledWith(2, '/util/fileurls', { objects: objects2 })
+ expect(objectUrls1).toEqual(mockResponse1.objectUrls)
+ expect(objectUrls2).toEqual(mockResponse2.objectUrls)
+ })
+
+ it('should batch requests when immediate flag is false', async () => {
+ const mockResponse = {
+ objectUrls: {
+ 'kodo://bucket/key1': 'https://bucket.example.com/key1',
+ 'kodo://bucket/key2': 'https://bucket.example.com/key2',
+ 'kodo://bucket/key3': 'https://bucket.example.com/key3'
+ } as UniversalToWebUrlMap
+ }
+ vi.mocked(client.post).mockResolvedValue(mockResponse)
+
+ const objects1 = ['kodo://bucket/key1', 'kodo://bucket/key2']
+ const objects2 = ['kodo://bucket/key3']
+ const promise1 = makeObjectUrls(objects1)
+ const promise2 = makeObjectUrls(objects2)
+ vi.advanceTimersByTime(2)
+ const [objectUrls1, objectUrls2] = await Promise.all([promise1, promise2])
+
+ expect(client.post).toHaveBeenCalledTimes(1)
+ expect(client.post).toHaveBeenCalledWith('/util/fileurls', {
+ objects: [...objects1, ...objects2]
+ })
+ expect(objectUrls1).toEqual(
+ Object.fromEntries(objects1.map((key) => [key, mockResponse.objectUrls[key]]))
+ )
+ expect(objectUrls2).toEqual(
+ Object.fromEntries(objects2.map((key) => [key, mockResponse.objectUrls[key]]))
+ )
+ })
+
+ it('should handle errors correctly', async () => {
+ const error = new Error('network error')
+ vi.mocked(client.post).mockRejectedValue(error)
+
+ const objects = ['kodo://bucket/key']
+ await expect(makeObjectUrls(objects, true)).rejects.toThrow(error)
+ })
+})

spx-gui/src/apis/util.ts

@@ -36,9 +36,31 @@ export function getUpInfo() {
  return client.get('/util/upinfo') as Promise<UpInfo>
 }
 
-export async function makeObjectUrls(objects: UniversalUrl[]) {
- const res = (await client.post('/util/fileurls', { objects: objects })) as {
- objectUrls: UniversalToWebUrlMap
+export const makeObjectUrls = (() => {
+ const make = async (objects: UniversalUrl[]) => {
+ const res = (await client.post('/util/fileurls', { objects: objects })) as {
+ objectUrls: UniversalToWebUrlMap
+ }
+ return res.objectUrls
  }
- return res.objectUrls
-}
+
+ const batch = new Set<UniversalUrl>()
+ let batchPromise: Promise<UniversalToWebUrlMap> | null = null
+ const processBatch = () => {
+ const currentBatch = Array.from(batch)
+ batch.clear()
+ batchPromise = null
+ return make(currentBatch)
+ }
+
+ return async (objects: UniversalUrl[], immediate = false): Promise<UniversalToWebUrlMap> => {
+ if (immediate) return make(objects)
+
+ objects.forEach((url) => batch.add(url))
+ if (batchPromise == null) {
+ batchPromise = new Promise((resolve) => setTimeout(() => resolve(processBatch()), 15))
+ }
+ const objectUrls = await batchPromise
+ return Object.fromEntries(objects.map((url) => [url, objectUrls[url]]))
+ }
+})()

spx-gui/src/models/common/cloud.ts

@@ -1,6 +1,6 @@
 import * as qiniu from 'qiniu-js'
 import { filename } from '@/utils/path'
-import type { WebUrl, UniversalUrl, FileCollection, UniversalToWebUrlMap } from '@/apis/common'
+import type { WebUrl, UniversalUrl, FileCollection } from '@/apis/common'
 import type { ProjectData } from '@/apis/project'
 import { IsPublic, addProject, getProject, updateProject } from '@/apis/project'
 import { getUpInfo as getRawUpInfo, makeObjectUrls, type UpInfo as RawUpInfo } from '@/apis/util'
@@ -57,24 +57,10 @@ export async function saveFiles(
 }
 
 export async function getFiles(fileCollection: FileCollection): Promise<Files> {
- const objectUniversalUrls = Object.values(fileCollection).filter(
- (url) => new URL(url).protocol === fileUniversalUrlSchemes.kodo
- )
- const objectUrls: UniversalToWebUrlMap = objectUniversalUrls.length
- ? await makeObjectUrls(objectUniversalUrls)
- : {}
-
  const files: Files = {}
  Object.keys(fileCollection).forEach((path) => {
  const universalUrl = fileCollection[path]
- let webUrl = universalUrl
-
- const objectUrl = objectUrls[universalUrl]
- if (objectUrl) {
- webUrl = objectUrl
- }
-
- const file = createFileWithWebUrl(webUrl, filename(path))
+ const file = createFileWithUniversalUrl(universalUrl, filename(path))
  setUniversalUrl(file, universalUrl)
  files[path] = file
  })
@@ -92,11 +78,18 @@ function getUniversalUrl(file: File): UniversalUrl | null {
  return file.meta.universalUrl ?? null
 }
 
-export function createFileWithWebUrl(webUrl: WebUrl, name = filename(webUrl)) {
+function createFileWithUniversalUrl(url: UniversalUrl, name = filename(url)) {
  return new File(name, async () => {
+ const webUrl = await universalUrlToWebUrl(url)
  const resp = await fetch(webUrl)
- const blob = await resp.blob()
- return blob.arrayBuffer()
+ return resp.arrayBuffer()
+ })
+}
+
+export function createFileWithWebUrl(url: WebUrl, name = filename(url)) {
+ return new File(name, async () => {
+ const resp = await fetch(url)
+ return resp.arrayBuffer()
  })
 }