From e4d81e50d7a7ee596fdd0c021e1eae596b8e2e4a Mon Sep 17 00:00:00 2001 From: wayblink Date: Tue, 18 Jul 2023 17:49:36 +0800 Subject: [PATCH] Add a full_meta.json store the complete metadata Signed-off-by: wayblink --- core/backup_impl_create_backup.go | 1 + core/backup_meta.go | 11 +++++++++++ example/db_support/prepare_data.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/core/backup_impl_create_backup.go b/core/backup_impl_create_backup.go index c752dc6..6918da3 100644 --- a/core/backup_impl_create_backup.go +++ b/core/backup_impl_create_backup.go @@ -570,6 +570,7 @@ func (b BackupContext) executeCreateBackup(ctx context.Context, request *backupp b.getStorageClient().Write(ctx, b.backupBucketName, CollectionMetaPath(b.backupRootPath, backupInfo.GetName()), output.CollectionMetaBytes) b.getStorageClient().Write(ctx, b.backupBucketName, PartitionMetaPath(b.backupRootPath, backupInfo.GetName()), output.PartitionMetaBytes) b.getStorageClient().Write(ctx, b.backupBucketName, SegmentMetaPath(b.backupRootPath, backupInfo.GetName()), output.SegmentMetaBytes) + b.getStorageClient().Write(ctx, b.backupBucketName, FullMetaPath(b.backupRootPath, backupInfo.GetName()), output.FullMetaBytes) log.Info("finish executeCreateBackup", zap.String("requestId", request.GetRequestId()), diff --git a/core/backup_meta.go b/core/backup_meta.go index 6808692..2052680 100644 --- a/core/backup_meta.go +++ b/core/backup_meta.go @@ -14,6 +14,7 @@ const ( COLLECTION_META_FILE = "collection_meta.json" PARTITION_META_FILE = "partition_meta.json" SEGMENT_META_FILE = "segment_meta.json" + FULL_META_FILE = "full_meta.json" SEPERATOR = "/" BINGLOG_DIR = "binlogs" @@ -32,6 +33,7 @@ type BackupMetaBytes struct { CollectionMetaBytes []byte PartitionMetaBytes []byte SegmentMetaBytes []byte + FullMetaBytes []byte } type LeveledBackupInfo struct { @@ -130,12 +132,17 @@ func serialize(backup *backuppb.BackupInfo) (*BackupMetaBytes, error) { if err != nil { return nil, err } + fullMetaBytes, err := json.Marshal(backup) + if err != nil { + return nil, err + } return &BackupMetaBytes{ BackupMetaBytes: backupMetaBytes, CollectionMetaBytes: collectionBackupMetaBytes, PartitionMetaBytes: partitionBackupMetaBytes, SegmentMetaBytes: segmentBackupMetaBytes, + FullMetaBytes: fullMetaBytes, }, nil } @@ -229,6 +236,10 @@ func SegmentMetaPath(backupRootPath, backupName string) string { return BackupMetaDirPath(backupRootPath, backupName) + SEPERATOR + SEGMENT_META_FILE } +func FullMetaPath(backupRootPath, backupName string) string { + return BackupMetaDirPath(backupRootPath, backupName) + SEPERATOR + FULL_META_FILE +} + func BackupBinlogDirPath(backupRootPath, backupName string) string { return backupRootPath + SEPERATOR + backupName + SEPERATOR + BINGLOG_DIR } diff --git a/example/db_support/prepare_data.py b/example/db_support/prepare_data.py index 75b0cf1..9f8e817 100644 --- a/example/db_support/prepare_data.py +++ b/example/db_support/prepare_data.py @@ -127,7 +127,7 @@ insert_result2 = hello_milvus2.insert(entities2) hello_milvus2.flush() -index_params2 = {"index_type": "TRIE"} +index_params2 = {"index_type": "Trie"} hello_milvus2.create_index("var", index_params2) print(f"Number of entities in hello_milvus2: {hello_milvus2.num_entities}") # check the num_entites