Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix backup check tool #6997

Merged
merged 1 commit into from
Oct 21, 2024
Merged

Fix backup check tool #6997

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 15 additions & 15 deletions common/config/rush/pnpm-lock.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions server/backup-service/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"@types/tar-stream": "^2.2.2",
"@types/tar-stream": "^3.1.3",
"@types/node": "~20.11.16",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
Expand All @@ -46,7 +46,7 @@
"@hcengineering/client-resources": "^0.6.27",
"@hcengineering/client": "^0.6.18",
"@hcengineering/model": "^0.6.11",
"tar-stream": "^2.2.0",
"tar-stream": "^3.1.7",
"@hcengineering/server-tool": "^0.6.0",
"@hcengineering/server-core": "^0.6.1",
"@hcengineering/server-storage": "^0.6.0",
Expand Down
4 changes: 2 additions & 2 deletions server/backup/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
"eslint-config-standard-with-typescript": "^40.0.0",
"prettier": "^3.1.0",
"typescript": "^5.3.3",
"@types/tar-stream": "^2.2.2",
"@types/tar-stream": "^3.1.3",
"@types/node": "~20.11.16",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
Expand All @@ -47,7 +47,7 @@
"@hcengineering/client": "^0.6.18",
"@hcengineering/model": "^0.6.11",
"@hcengineering/analytics": "^0.6.0",
"tar-stream": "^2.2.0",
"tar-stream": "^3.1.7",
"@hcengineering/server-tool": "^0.6.0",
"@hcengineering/server-client": "^0.6.0",
"@hcengineering/server-token": "^0.6.11",
Expand Down
35 changes: 19 additions & 16 deletions server/backup/src/backup.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ import { type StorageAdapter } from '@hcengineering/server-core'
import { fullTextPushStagePrefix } from '@hcengineering/server-indexer'
import { generateToken } from '@hcengineering/server-token'
import { connect } from '@hcengineering/server-tool'
import { createReadStream, createWriteStream, existsSync, mkdirSync } from 'node:fs'
import { createReadStream, createWriteStream, existsSync, mkdirSync, statSync } from 'node:fs'
import { rm } from 'node:fs/promises'
import { basename, dirname } from 'node:path'
import { PassThrough } from 'node:stream'
Expand Down Expand Up @@ -178,7 +178,7 @@ async function loadDigest (
result.delete(k as Ref<Doc>)
}
} catch (err: any) {
ctx.error('digest is broken, will do full backup for', { domain })
ctx.error('digest is broken, will do full backup for', { domain, err: err.message, snapshot })
}
}
// Stop if stop date is matched and provided
Expand Down Expand Up @@ -236,14 +236,10 @@ async function verifyDigest (
blobs.set(bname, { doc, buffer: undefined })
} else {
blobs.delete(bname)
const blob = doc as Blob

if (blob.size === bf.length) {
validDocs.add(name as Ref<Doc>)
}
validDocs.add(bname as Ref<Doc>)
}
} else {
validDocs.add(name as Ref<Doc>)
validDocs.add(bname as Ref<Doc>)
}
next()
})
Expand All @@ -265,10 +261,7 @@ async function verifyDigest (
sz = bf.length
}

// If blob size matches doc size, remove from requiredDocs
if (sz === bf.length) {
validDocs.add(name as Ref<Doc>)
}
validDocs.add(name as Ref<Doc>)
}
next()
})
Expand Down Expand Up @@ -364,7 +357,7 @@ async function verifyDigest (
}
} catch (err: any) {
digestToRemove.add(snapshot)
ctx.error('digest is broken, will do full backup for', { domain })
ctx.error('digest is broken, will do full backup for', { domain, err: err.message, snapshot })
modified = true
}
}
Expand Down Expand Up @@ -1490,6 +1483,7 @@ export async function backupSize (storage: BackupStorage): Promise<void> {
*/
export async function backupDownload (storage: BackupStorage, storeIn: string): Promise<void> {
const infoFile = 'backup.json.gz'
const sizeFile = 'backup.size.gz'

if (!(await storage.exists(infoFile))) {
throw new Error(`${infoFile} should present to restore`)
Expand All @@ -1499,15 +1493,24 @@ export async function backupDownload (storage: BackupStorage, storeIn: string):
const backupInfo: BackupInfo = JSON.parse(gunzipSync(await storage.loadFile(infoFile)).toString())
console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)

let sizeInfo: Record<string, number> = {}
if (await storage.exists(sizeFile)) {
sizeInfo = JSON.parse(gunzipSync(await storage.loadFile(sizeFile)).toString())
}
console.log('workspace:', backupInfo.workspace ?? '', backupInfo.version)

const addFileSize = async (file: string | undefined | null, force: boolean = false): Promise<void> => {
if (file != null) {
const target = join(storeIn, file)
const dir = dirname(target)
if (!existsSync(dir)) {
mkdirSync(dir, { recursive: true })
}
if (!existsSync(target) || force) {
const fileSize = await storage.stat(file)

const serverSize: number | undefined = sizeInfo[file]

if (!existsSync(target) || force || (serverSize !== undefined && serverSize !== statSync(target).size)) {
const fileSize = serverSize ?? (await storage.stat(file))
console.log('downloading', file, fileSize)
const readStream = await storage.load(file)
const outp = createWriteStream(target)
Expand Down Expand Up @@ -1781,7 +1784,7 @@ export async function restore (

if (sendSize > dataUploadSize || (doc === undefined && docs.length > 0)) {
totalSend += docs.length
ctx.info('upload', {
ctx.info('upload-' + c, {
docs: docs.length,
totalSend,
from: docsToAdd.size + totalSend,
Expand Down