feat: new snapshot format

packages/core/src/snapshot/addResourcesToSnapshot.ts

@@ -1,11 +1,12 @@
 import fs from 'node:fs'
 import path from 'pathe'
+import SuperJSON from 'superjson'
 import type { DatabaseSnapshot } from '../types/snapshot.js'
 import { getCurrentBranchFolder } from '../resource/branch.js'
-import { ensureDir } from '../util/fs.js'
 import { getResourceInstanceStorage } from '../resource/storage.js'
 import type { MoquerieInstance } from '../instance.js'
 import { getSnapshotFolder } from './folder.js'
+import { migrateSnapshotFolder } from './migrate.js'
 
 export interface AddResourcesToSnapshotOptions {
   snapshot: DatabaseSnapshot
@@ -15,21 +16,29 @@ export interface AddResourcesToSnapshotOptions {
 export async function addResourcesToSnapshot(mq: MoquerieInstance, options: AddResourcesToSnapshotOptions) {
   const { snapshot, resourceIds } = options
 
-  // Copy resources
   const snapshotFolder = await getSnapshotFolder(mq, snapshot)
+
+  await migrateSnapshotFolder(snapshotFolder)
+
   for (const resourceName in resourceIds) {
     const sourceFolder = path.join(getCurrentBranchFolder(mq), resourceName)
-    const targetFolder = path.join(snapshotFolder, resourceName)
-    await ensureDir(targetFolder)
+
+    const targetFile = path.join(snapshotFolder, `${resourceName}.res.json`)
 
     const resourceStorage = await getResourceInstanceStorage(mq, resourceName)
 
     const ids = resourceIds[resourceName]
+    const data: Record<string, any> = SuperJSON.parse(await fs.promises.readFile(targetFile, 'utf8'))
+
     for (const id of ids) {
       const file = resourceStorage.manifest.files[id]
       const sourceFile = path.join(sourceFolder, file)
-      const targetFile = path.join(targetFolder, `${id}.json`)
-      await fs.promises.copyFile(sourceFile, targetFile)
+      const content = await fs.promises.readFile(sourceFile, 'utf8')
+      data[id] = SuperJSON.parse(content)
     }
+
+    // Sort by key to ensure consistent order
+    const sortedData = Object.fromEntries(Object.entries(data).sort(([a], [b]) => a.localeCompare(b)))
+    await fs.promises.writeFile(targetFile, JSON.stringify(SuperJSON.serialize(sortedData), null, 2), 'utf8')
   }
 }

packages/core/src/snapshot/createSnapshot.ts

@@ -1,12 +1,13 @@
 import fs from 'node:fs'
 import path from 'pathe'
+import SuperJSON from 'superjson'
 import type { DBLocation } from '../types/db.js'
 import { getCurrentUser } from '../user/getCurrentUser.js'
 import type { DatabaseSnapshot } from '../types/snapshot.js'
-import { ensureDir } from '../util/fs.js'
 import { getCurrentBranchFolder } from '../resource/branch.js'
 import { getResourceInstanceStorage } from '../resource/storage.js'
 import type { MoquerieInstance } from '../instance.js'
+import { ensureDir } from '../util/fs.js'
 import { getSnapshotFolder } from './folder.js'
 import { getSnapshotStorage } from './storage.js'
 
@@ -48,21 +49,27 @@ export async function createSnapshot(mq: MoquerieInstance, options: CreateSnapsh
 
   // Copy resources
   const snapshotFolder = await getSnapshotFolder(mq, snapshotItem)
+  await ensureDir(snapshotFolder)
+
   for (const resourceName in resourceIds) {
-    if (resourceIds[resourceName].length) {
+    const ids = resourceIds[resourceName]
+    if (ids.length) {
+      const data: { [id: string]: any } = {}
       const sourceFolder = path.join(getCurrentBranchFolder(mq), resourceName)
-      const targetFolder = path.join(snapshotFolder, resourceName)
-      await ensureDir(targetFolder)
+      const targetFile = path.join(snapshotFolder, `${resourceName}.res.json`)
 
       const resourceStorage = await getResourceInstanceStorage(mq, resourceName)
 
-      const ids = resourceIds[resourceName]
       for (const id of ids) {
         const file = resourceStorage.manifest.files[id]
         const sourceFile = path.join(sourceFolder, file)
-        const targetFile = path.join(targetFolder, `${id}.json`)
-        await fs.promises.copyFile(sourceFile, targetFile)
+        const content = await fs.promises.readFile(sourceFile, 'utf8')
+        data[id] = SuperJSON.parse(content)
       }
+
+      // Sort by key to ensure consistent order
+      const sortedData = Object.fromEntries(Object.entries(data).sort(([a], [b]) => a.localeCompare(b)))
+      await fs.promises.writeFile(targetFile, JSON.stringify(SuperJSON.serialize(sortedData), null, 2), 'utf8')
     }
   }
 

packages/core/src/snapshot/folder.ts

@@ -3,7 +3,7 @@ import type { DatabaseSnapshot } from '../types/snapshot.js'
 import type { MoquerieInstance } from '../instance.js'
 import { getSnapshotStorage } from './storage.js'
 
-export async function getSnapshotFolder(mq: MoquerieInstance, snapshot: DatabaseSnapshot) {
+export async function getSnapshotFolder(mq: MoquerieInstance, snapshot: Pick<DatabaseSnapshot, 'id' | 'location'>) {
   const storage = await getSnapshotStorage(mq)
   return path.join(storage[snapshot.location].folder, snapshot.id)
 }

packages/core/src/snapshot/migrate.ts

@@ -0,0 +1,46 @@
+import fs from 'node:fs'
+import path from 'pathe'
+import glob from 'fast-glob'
+import SuperJSON from 'superjson'
+
+/**
+ * Migrate snapshot folder to new format with each resource type in a single file
+ * @param snapshotFolder The snapshot folder to migrate
+ * @returns Whether the folder was migrated
+ */
+export async function migrateSnapshotFolder(snapshotFolder: string) {
+  const files = await glob('*/*.json', {
+    cwd: snapshotFolder,
+    onlyFiles: true,
+  })
+
+  if (files.length === 0) {
+    return false
+  }
+
+  const dataPerResource = new Map<string, Record<string, any>>()
+
+  for (const file of files) {
+    const [resourceName, id] = file.split('/')
+    const filePath = path.join(snapshotFolder, file)
+    const content = await fs.promises.readFile(filePath, 'utf8')
+    const fileData = SuperJSON.parse(content)
+    let dataForResource = dataPerResource.get(resourceName)
+    if (!dataForResource) {
+      dataForResource = {}
+      dataPerResource.set(resourceName, dataForResource)
+    }
+    dataForResource[id] = fileData
+  }
+
+  for (const [resourceName, data] of dataPerResource) {
+    const resourceFile = path.join(snapshotFolder, `${resourceName}.res.json`)
+    // Sort by key to ensure consistent order
+    const sortedData = Object.fromEntries(Object.entries(data).sort(([a], [b]) => a.localeCompare(b)))
+    await fs.promises.writeFile(resourceFile, JSON.stringify(SuperJSON.serialize(sortedData), null, 2), 'utf8')
+  }
+
+  await Promise.all(files.map(file => fs.promises.rm(path.join(snapshotFolder, file))))
+
+  return true
+}

packages/core/src/snapshot/readResources.ts

@@ -1,29 +1,60 @@
 import fs from 'node:fs'
 import path from 'pathe'
 import SuperJSON from 'superjson'
+import glob from 'fast-glob'
 import type { DatabaseSnapshot } from '../types/snapshot.js'
 import type { ResourceInstance } from '../types/resource.js'
 import type { MoquerieInstance } from '../instance.js'
 import { getSnapshotFolder } from './folder.js'
+import { migrateSnapshotFolder } from './migrate.js'
 
 export async function readSnapshotResourceIds(mq: MoquerieInstance, snapshot: DatabaseSnapshot) {
   const folder = await getSnapshotFolder(mq, snapshot)
-  const resourceTypeFolders = (await fs.promises.readdir(folder)).filter(file => fs.statSync(path.join(folder, file)).isDirectory())
 
   const result: { [resourceName: string]: string[] } = {}
 
-  await Promise.all(resourceTypeFolders.map(async (resourceName) => {
-    const resourceFolder = path.join(folder, resourceName)
-    const resourceFiles = await fs.promises.readdir(resourceFolder)
-    const resourceIds = resourceFiles.map(file => path.basename(file, path.extname(file)))
-    result[resourceName] = resourceIds
-  }))
+  const resourceFiles = await glob('*.res.json', {
+    cwd: folder,
+    onlyFiles: true,
+  })
+  if (resourceFiles.length) {
+    // New format with each resource type in a single file
+
+    await Promise.all(resourceFiles.map(async (file) => {
+      const resourceName = path.basename(file, '.res.json')
+      const content = await fs.promises.readFile(path.join(folder, file), 'utf-8')
+      const data = SuperJSON.parse<any>(content)
+      result[resourceName] = Object.keys(data)
+    }))
+  }
+  else {
+    // Old format with files inside resource folders
+
+    const resourceTypeFolders = (await fs.promises.readdir(folder)).filter(file => fs.statSync(path.join(folder, file)).isDirectory())
+
+    await Promise.all(resourceTypeFolders.map(async (resourceName) => {
+      const resourceFolder = path.join(folder, resourceName)
+      const resourceFiles = await fs.promises.readdir(resourceFolder)
+      const resourceIds = resourceFiles.map(file => path.basename(file, path.extname(file)))
+      result[resourceName] = resourceIds
+    }))
+  }
 
   return result
 }
 
 export async function readSnapshotResources(mq: MoquerieInstance, snapshot: DatabaseSnapshot, resourceName: string): Promise<ResourceInstance[]> {
   const folder = await getSnapshotFolder(mq, snapshot)
+  const resourceFile = path.join(folder, `${resourceName}.res.json`)
+
+  // New format with each resource type in a single file
+  if (fs.existsSync(resourceFile)) {
+    const content = await fs.promises.readFile(resourceFile, 'utf-8')
+    const data = SuperJSON.parse<any>(content)
+    return Object.values<any>(data).map(({ item }) => item as ResourceInstance)
+  }
+
+  // Old format with files inside resource folders
   const resourceFolder = path.join(folder, resourceName)
   if (!fs.existsSync(resourceFolder)) {
     return []
@@ -38,16 +69,20 @@ export async function readSnapshotResources(mq: MoquerieInstance, snapshot: Data
 
 export async function readSnapshotAllResources(mq: MoquerieInstance, snapshot: DatabaseSnapshot) {
   const folder = await getSnapshotFolder(mq, snapshot)
-  const resourceTypeFolders = (await fs.promises.readdir(folder)).filter(file => fs.statSync(path.join(folder, file)).isDirectory())
+
+  await migrateSnapshotFolder(folder)
+
   const result: { [resourceName: string]: ResourceInstance[] } = {}
 
-  await Promise.all(resourceTypeFolders.map(async (resourceName) => {
-    const resourceFolder = path.join(folder, resourceName)
-    const resourceFiles = await fs.promises.readdir(resourceFolder)
-    result[resourceName] = await Promise.all(resourceFiles.map(async (file) => {
-      const content = await fs.promises.readFile(path.join(resourceFolder, file), 'utf-8')
-      return SuperJSON.parse<any>(content).item as ResourceInstance
-    }))
+  const resourceFiles = await glob('*.res.json', {
+    cwd: folder,
+    onlyFiles: true,
+  })
+  await Promise.all(resourceFiles.map(async (file) => {
+    const resourceName = path.basename(file, '.res.json')
+    const content = await fs.promises.readFile(path.join(folder, file), 'utf-8')
+    const data = SuperJSON.parse<any>(content)
+    result[resourceName] = Object.values<any>(data).map(({ item }) => item as ResourceInstance)
   }))
 
   return result

packages/core/src/snapshot/removeResourcesFromSnapshot.ts

@@ -1,8 +1,10 @@
 import fs from 'node:fs'
 import path from 'pathe'
+import SuperJSON from 'superjson'
 import type { DatabaseSnapshot } from '../types/snapshot.js'
 import type { MoquerieInstance } from '../instance.js'
 import { getSnapshotFolder } from './folder.js'
+import { migrateSnapshotFolder } from './migrate.js'
 
 export interface RemoveResourcesToSnapshotOptions {
   snapshot: DatabaseSnapshot
@@ -14,14 +16,24 @@ export async function removeResourcesFromSnapshot(mq: MoquerieInstance, options:
 
   // Delete resources
   const snapshotFolder = await getSnapshotFolder(mq, snapshot)
+  await migrateSnapshotFolder(snapshotFolder)
+
   for (const resourceName in resourceIds) {
-    const targetFolder = path.join(snapshotFolder, resourceName)
     const ids = resourceIds[resourceName]
-    for (const id of ids) {
-      const targetFile = path.join(targetFolder, `${id}.json`)
-      if (fs.existsSync(targetFile)) {
+    const targetFile = path.join(snapshotFolder, `${resourceName}.res.json`)
+    if (fs.existsSync(targetFile)) {
+      // New format with each resource type in a single file
+      const content = await fs.promises.readFile(targetFile, 'utf-8')
+      const data = SuperJSON.parse<any>(content)
+      for (const id of ids) {
+        delete data[id]
+      }
+      if (Object.keys(data).length === 0) {
         await fs.promises.rm(targetFile)
       }
+      else {
+        await fs.promises.writeFile(targetFile, JSON.stringify(SuperJSON.serialize(data), null, 2), 'utf-8')
+      }
     }
   }
 }