大幅优化备份速度和CPU使用率，修复首次手动备份时由于备份时间过长导致MC超时自动关闭

LingMuQingYu · Jun 20, 2024 · 2a08e38 · 2a08e38
1 parent d915b23
commit 2a08e38
Show file tree

Hide file tree

Showing 6 changed files with 85 additions and 26 deletions.
diff --git a/common.gradle b/common.gradle
@@ -66,6 +66,7 @@ dependencies {
     include(implementation('org.mongodb:mongodb-driver-core:4.11.1'))
     include(implementation('org.mongodb:bson:4.11.1'))
     include(implementation('com.h2database:h2:2.2.224'))
+    include(implementation('org.lz4:lz4-java:1.8.0'))
 
 }
 

diff --git a/src/main/java/dev/skydynamic/quickbackupmulti/command/MakeCommand.java b/src/main/java/dev/skydynamic/quickbackupmulti/command/MakeCommand.java
@@ -26,9 +26,10 @@ static class makeRunnable implements Runnable {
 
         @Override
         public void run() {
+            long l = System.currentTimeMillis();
             LOGGER.info("Make Backup thread started...");
             make(commandSource, name, desc);
-            LOGGER.info("Make Backup thread close");
+            LOGGER.info("Make Backup thread close => {}ms", System.currentTimeMillis() - l);
         }
     }
 
@@ -45,7 +46,8 @@ public void run() {
 //            .executes(it -> makeSaveBackup(it.getSource(), String.valueOf(System.currentTimeMillis()), StringArgumentType.getString(it, "desc"))));
 
     private static int makeSaveBackup(ServerCommandSource commandSource, String name, String desc) {
-        new makeRunnable(commandSource, name, desc).run();
+        Thread.startVirtualThread(new makeRunnable(commandSource, name, desc));
+//        new makeRunnable(commandSource, name, desc).run();
         return 1;
     }
 }
diff --git a/src/main/java/dev/skydynamic/quickbackupmulti/config/ConfigStorage.java b/src/main/java/dev/skydynamic/quickbackupmulti/config/ConfigStorage.java
@@ -1,18 +1,20 @@
 package dev.skydynamic.quickbackupmulti.config;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Objects;
 
 public class ConfigStorage {
     @Ignore
     public static final ConfigStorage DEFAULT = new ConfigStorage(
-        new ArrayList<>(),
-        "zh_cn",
-        false,
-        "* * 0/4 * * ?",
-        14400,
-        "interval",
-        true,
-        "mongodb://localhost:27017"
+            new ArrayList<>(),
+            "zh_cn",
+            false,
+            "* * 0/4 * * ?",
+            14400,
+            "interval",
+            true,
+            "mongodb://localhost:27017",
+            false
     );
 
     public ArrayList<String> ignoredFiles;
@@ -24,16 +26,18 @@ public class ConfigStorage {
 
     public boolean useInternalDataBase;
     public String mongoDBUri;
+    public boolean useFastHash;
 
     public ConfigStorage(
-        ArrayList<String> IgnoredFiles,
-        String lang,
-        boolean scheduleBackup,
-        String scheduleCron,
-        int scheduleInterval,
-        String scheduleMode,
-        boolean useInternalDataBase,
-        String mongoDBUri) {
+            ArrayList<String> IgnoredFiles,
+            String lang,
+            boolean scheduleBackup,
+            String scheduleCron,
+            int scheduleInterval,
+            String scheduleMode,
+            boolean useInternalDataBase,
+            String mongoDBUri,
+            boolean useFastHash) {
         this.ignoredFiles = IgnoredFiles;
         this.lang = lang;
         this.scheduleBackup = scheduleBackup;
@@ -42,6 +46,7 @@ public ConfigStorage(
         this.scheduleMode = scheduleMode;
         this.useInternalDataBase = useInternalDataBase;
         this.mongoDBUri = mongoDBUri;
+        this.useFastHash = useFastHash;
     }
 
     public boolean getScheduleBackup() {

diff --git a/src/main/java/dev/skydynamic/quickbackupmulti/config/QuickBackupMultiConfig.java b/src/main/java/dev/skydynamic/quickbackupmulti/config/QuickBackupMultiConfig.java
@@ -141,6 +141,12 @@ public String getMongoDBUri() {
         }
     }
 
+    public boolean getUseFastHash() {
+        synchronized (lock) {
+            return configStorage.useFastHash;
+        }
+    }
+
     public void setLang(String lang) {
         synchronized (lock) {
             configStorage.lang = lang;

diff --git a/src/main/java/dev/skydynamic/quickbackupmulti/utils/MakeUtils.java b/src/main/java/dev/skydynamic/quickbackupmulti/utils/MakeUtils.java
@@ -87,7 +87,7 @@ private static <T extends HashMap<String, String>> HashMap<String, Object> compa
         List<String> indexBackupList // 索引列表
     ) throws Exception {
         // 获取文件Hash
-        String fileHash = getFileHash(file.toPath());
+        String fileHash = getFileHash(file.toPath(), null);
         // 如果不是第一次备份
         if (!isFirstBackup) {
             // 对比文件Hash

diff --git a/src/main/java/dev/skydynamic/quickbackupmulti/utils/hash/HashUtils.java b/src/main/java/dev/skydynamic/quickbackupmulti/utils/hash/HashUtils.java
@@ -1,21 +1,58 @@
 package dev.skydynamic.quickbackupmulti.utils.hash;
 
+import dev.skydynamic.quickbackupmulti.config.Config;
 import dev.skydynamic.quickbackupmulti.storage.FileHashes;
+import net.jpountz.xxhash.XXHashFactory;
 
 import java.io.File;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.MessageDigest;
 
+import static dev.skydynamic.quickbackupmulti.QuickBackupMulti.LOGGER;
+
 public class HashUtils {
 
-    public static String getFileHash(Path file) throws Exception {
-        byte[] fileData = Files.readAllBytes(file);
+    private static final String SHA_256 = "SHA-256";
+    private final static XXHashFactory XX_HASH_FACTORY = XXHashFactory.fastestInstance();
+    private static Boolean useFastHash;
 
-        MessageDigest md = MessageDigest.getInstance("SHA-256");
+    public static String getFileHash(Path file, String algorithm) throws Exception {
+        if (useFastHash == null) {
+            useFastHash = Config.INSTANCE.getUseFastHash();
+            LOGGER.info("Backup UseFastHash => {}", useFastHash);
+        }
+        if (algorithm == null) {
+            algorithm = SHA_256;
+        }
+        byte[] fileData;
+        byte[] digest;
+        if (useFastHash) {
+            String fileName = file.getFileName().toString();
+            boolean isAnvil = fileName.startsWith("r.") && fileName.endsWith(".mca");
+            if (isAnvil) {
+                /*
+                   对于(Anvil)区块文件来说，只需要校验8kB文件头即可得知区块文件是否有变动
+                   区块位置：0 - 4095 每个区块占用4个字节，存储区块位置，未生成则为0填充
+                   区块更新时间戳：4096 - 8191 每个区块占用4个字节，存储区块更新的时间戳
+                 */
+                try (InputStream is = Files.newInputStream(file)) {
+                    fileData = new byte[8192];
+                    is.read(fileData, 0, fileData.length);
+                }
+            } else {
+                fileData = Files.readAllBytes(file);
+            }
+            // 使用高性能非加密哈希算法
+            digest = longToBytes(XX_HASH_FACTORY.hash64().hash(fileData, 0, fileData.length, 0));
+            return bytesToHex(digest);
+        }
+        fileData = Files.readAllBytes(file);
+        MessageDigest md = MessageDigest.getInstance(algorithm);
         md.update(fileData);
-        byte[] digest = md.digest();
-
+        digest = md.digest();
         return bytesToHex(digest);
     }
 
@@ -28,11 +65,19 @@ private static String bytesToHex(byte[] hash) {
         StringBuilder hexString = new StringBuilder(2 * hash.length);
         for (byte b : hash) {
             String hex = Integer.toHexString(0xff & b);
-            if(hex.length() == 1) {
+            if (hex.length() == 1) {
                 hexString.append('0');
             }
             hexString.append(hex);
         }
         return hexString.toString();
     }
+
+    public static byte[] longToBytes(long l) {
+        //分配缓冲区，单位为字节，一个long类型占8个字节，所以分配为8
+        ByteBuffer byteBuffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE);
+        //参数一为起始位置（不指定默认为0），参数二为所放的值
+        byteBuffer.putLong(0, l);
+        return byteBuffer.array();
+    }
 }