From bae4258a396ae37b20cd5388b602fc7f228651ff Mon Sep 17 00:00:00 2001 From: EmmyMiao87 <522274284@qq.com> Date: Thu, 20 May 2021 13:39:33 +0800 Subject: [PATCH] [Bug-fix][Backup] Modify the persistence logic of backup and restore The expose annotation is used in the persistence logic used by the old backup recovery. This annotation by itself is meant to ignore some variables when serializing and deserializing. However, this variable was used incorrectly and gson did not ignore the variables that should have been ignored. This results in duplicate initialization when fe is restarted. This pr uses the doris wrapped Gson directly, than eliminates the use of the expose annotation. Fixed sortedTabletInfoList being repeatedly initialized resulting in incorrect numbers. Fixed #5852 --- .../apache/doris/backup/BackupJobInfo.java | 39 ++++++++++++++----- .../apache/doris/persist/gson/GsonUtils.java | 4 ++ .../doris/backup/BackupJobInfoTest.java | 28 ++++++++++--- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java index 072f01aa4d61ff..4c7f6f52a841c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java @@ -35,6 +35,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; import com.google.common.base.Joiner; import com.google.common.collect.Lists; @@ -59,8 +60,6 @@ import java.util.Set; import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; import org.glassfish.jersey.internal.guava.Sets; @@ -72,6 +71,7 @@ public class BackupJobInfo implements Writable { private static final Logger LOG = LogManager.getLogger(BackupJobInfo.class); + @SerializedName("name") public String name; @SerializedName("database") public String dbName; @@ -87,7 +87,7 @@ public class BackupJobInfo implements Writable { // include other objects: view, external table @SerializedName("new_backup_objects") public NewBackupObjects newBackupObjects = new NewBackupObjects(); - public boolean success = true; + public boolean success; @SerializedName("backup_result") public String successJson = "succeed"; @@ -96,7 +96,6 @@ public class BackupJobInfo implements Writable { // This map is used to save the table alias mapping info when processing a restore job. // origin -> alias - @Expose(serialize = false, deserialize = false) public Map tblAlias = Maps.newHashMap(); public void initBackupJobInfoAfterDeserialize() { @@ -274,10 +273,13 @@ public String getOrginNameByAlias(String alias) { } public static class BriefBackupJobInfo { + @SerializedName("name") public String name; + @SerializedName("database") public String database; @SerializedName("backup_time") public long backupTime; + @SerializedName("content") public BackupContent content; @SerializedName("olap_table_list") public List olapTableList = Lists.newArrayList(); @@ -309,12 +311,14 @@ public static BriefBackupJobInfo fromBackupJobInfo(BackupJobInfo backupJobInfo) } public static class BriefBackupOlapTable { + @SerializedName("name") public String name; @SerializedName("partition_names") public List partitionNames; } public static class NewBackupObjects { + @SerializedName("views") public List views = Lists.newArrayList(); @SerializedName("odbc_tables") public List odbcTables = Lists.newArrayList(); @@ -323,7 +327,9 @@ public static class NewBackupObjects { } public static class BackupOlapTableInfo { + @SerializedName("id") public long id; + @SerializedName("partitions") public Map partitions = Maps.newHashMap(); public boolean containsPart(String partName) { @@ -349,10 +355,13 @@ public void retainPartitions(Collection partNames) { } public static class BackupPartitionInfo { + @SerializedName("id") public long id; + @SerializedName("version") public long version; @SerializedName("version_hash") public long versionHash; + @SerializedName("indexes") public Map indexes = Maps.newHashMap(); public BackupIndexInfo getIdx(String idxName) { @@ -361,13 +370,14 @@ public BackupIndexInfo getIdx(String idxName) { } public static class BackupIndexInfo { + @SerializedName("id") public long id; @SerializedName("schema_hash") public int schemaHash; + @SerializedName("tablets") public Map> tablets = Maps.newHashMap(); @SerializedName("tablets_order") public List tabletsOrder = Lists.newArrayList(); - @Expose(serialize = false, deserialize = false) public List sortedTabletInfoList = Lists.newArrayList(); public List getTabletFiles(long tabletId) { @@ -388,7 +398,9 @@ private List getSortedTabletIds() { } public static class BackupTabletInfo { + @SerializedName("id") public long id; + @SerializedName("files") public List files; public BackupTabletInfo(long id, List files) { @@ -398,11 +410,14 @@ public BackupTabletInfo(long id, List files) { } public static class BackupViewInfo { + @SerializedName("id") public long id; + @SerializedName("name") public String name; } public static class BackupOdbcTableInfo { + @SerializedName("id") public long id; @SerializedName("doris_table_name") public String dorisTableName; @@ -412,15 +427,20 @@ public static class BackupOdbcTableInfo { public String linkedOdbcTableName; @SerializedName("resource_name") public String resourceName; + @SerializedName("host") public String host; + @SerializedName("port") public String port; + @SerializedName("user") public String user; + @SerializedName("driver") public String driver; @SerializedName("odbc_type") public String odbcType; } public static class BackupOdbcResourceInfo { + @SerializedName("name") public String name; } @@ -624,8 +644,7 @@ private static BackupJobInfo genFromJson(String json) { * } * } */ - Gson gson = new Gson(); - BackupJobInfo jobInfo = gson.fromJson(json, BackupJobInfo.class); + BackupJobInfo jobInfo = GsonUtils.GSON.fromJson(json, BackupJobInfo.class); jobInfo.initBackupJobInfoAfterDeserialize(); return jobInfo; } @@ -644,16 +663,16 @@ public void writeToFile(File jobInfoFile) throws FileNotFoundException { // Only return basic info, table and partitions public String getBrief() { BriefBackupJobInfo briefBackupJobInfo = BriefBackupJobInfo.fromBackupJobInfo(this); - Gson gson = new GsonBuilder().setPrettyPrinting().create(); + Gson gson = GsonUtils.GSON_PRETTY_PRINTING; return gson.toJson(briefBackupJobInfo); } public String toJson(boolean prettyPrinting) { Gson gson; if (prettyPrinting) { - gson = new GsonBuilder().setPrettyPrinting().create(); + gson = GsonUtils.GSON_PRETTY_PRINTING; } else { - gson = new Gson(); + gson = GsonUtils.GSON; } return gson.toJson(this); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index e5b86f514a20b4..15436cf64de3d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -132,9 +132,13 @@ public class GsonUtils { .registerTypeAdapter(ImmutableMap.class, new ImmutableMapDeserializer()) .registerTypeAdapter(AtomicBoolean.class, new AtomicBooleanAdapter()); + private static final GsonBuilder GSON_BUILDER_PRETTY_PRINTING = GSON_BUILDER.setPrettyPrinting(); + // this instance is thread-safe. public static final Gson GSON = GSON_BUILDER.create(); + public static final Gson GSON_PRETTY_PRINTING = GSON_BUILDER_PRETTY_PRINTING.create(); + /* * The exclusion strategy of GSON serialization. * Any fields without "@SerializedName" annotation with be ignore with diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobInfoTest.java index b0beb30c430e2d..eeec431b1e3a5f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobInfoTest.java @@ -17,11 +17,6 @@ package org.apache.doris.backup; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; @@ -31,6 +26,11 @@ import java.io.IOException; import java.io.PrintWriter; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + public class BackupJobInfoTest { private static String fileName = "job_info.txt"; @@ -169,6 +169,7 @@ public void testReadWrite() { Assert.assertEquals("view1", jobInfo.newBackupObjects.views.get(0).name); File tmpFile = new File("./tmp"); + File tmpFile1 = new File("./tmp1"); try { DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpFile)); jobInfo.write(out); @@ -186,11 +187,28 @@ public void testReadWrite() { Assert.assertEquals(jobInfo.newBackupObjects.views.size(), newInfo.newBackupObjects.views.size()); Assert.assertEquals("view1", newInfo.newBackupObjects.views.get(0).name); + out = new DataOutputStream(new FileOutputStream(tmpFile1)); + newInfo.write(out); + out.flush(); + out.close(); + + in = new DataInputStream(new FileInputStream(tmpFile1)); + BackupJobInfo newInfo1 = BackupJobInfo.read(in); + in.close(); + + Assert.assertEquals( + newInfo.backupOlapTableObjects.get("table2").getPartInfo("partition1") + .indexes.get("table2").sortedTabletInfoList.size(), + newInfo1.backupOlapTableObjects.get("table2").getPartInfo("partition1") + .indexes.get("table2").sortedTabletInfoList.size()); + } catch (IOException e) { e.printStackTrace(); Assert.fail(); } finally { tmpFile.delete(); + tmpFile1.delete(); } + } }