diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java index 5e3b6febf5..a627b73b79 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java @@ -360,7 +360,7 @@ public void truncateBackend() { * When restarting, load the snapshot first and then read backend, * will not encounter such an intermediate state. */ - this.storeProvider.writeSnapshot(); + this.storeProvider.createSnapshot(); } finally { LockUtil.unlock(this.name, LockUtil.GRAPH_LOCK); } @@ -371,7 +371,7 @@ public void truncateBackend() { public void createSnapshot() { LockUtil.lock(this.name, LockUtil.GRAPH_LOCK); try { - this.storeProvider.writeSnapshot(); + this.storeProvider.createSnapshot(); } finally { LockUtil.unlock(this.name, LockUtil.GRAPH_LOCK); } @@ -382,7 +382,7 @@ public void createSnapshot() { public void resumeSnapshot() { LockUtil.lock(this.name, LockUtil.GRAPH_LOCK); try { - this.storeProvider.readSnapshot(); + this.storeProvider.resumeSnapshot(); } finally { LockUtil.unlock(this.name, LockUtil.GRAPH_LOCK); } @@ -1445,7 +1445,7 @@ public void invalid2(HugeType type, Object[] ids) { @Override public void clear(HugeType type) { - this.hub.notify(Events.CACHE, Cache.ACTION_CLEAR, type, null); + this.hub.notify(Events.CACHE, Cache.ACTION_CLEAR, type); } @Override diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java index 918f4cdf91..48aeed09b8 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/AbstractBackendStoreProvider.java @@ -157,19 +157,23 @@ public void initSystemInfo(HugeGraph graph) { } @Override - public void writeSnapshot() { + public void createSnapshot() { + long begin = System.currentTimeMillis(); String snapshotPrefix = StoreSnapshotFile.SNAPSHOT_DIR; for (BackendStore store : this.stores.values()) { - store.writeSnapshot(snapshotPrefix); + store.createSnapshot(snapshotPrefix); } + + LOG.info("Create snapshot cost {}ms", + System.currentTimeMillis() - begin); } @Override - public void readSnapshot() { + public void resumeSnapshot() { String snapshotPrefix = StoreSnapshotFile.SNAPSHOT_DIR; Set snapshotDirs = new HashSet<>(); for (BackendStore store : this.stores.values()) { - snapshotDirs.addAll(store.readSnapshot(snapshotPrefix)); + snapshotDirs.addAll(store.resumeSnapshot(snapshotPrefix)); } // Delete all snapshot parent directories for (String snapshotDir : snapshotDirs) { diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStore.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStore.java index 914f3f5573..42257fcdbb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStore.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStore.java @@ -117,11 +117,11 @@ public default void setCounterLowest(HugeType type, long lowest) { // Get current counter for a specific type public long getCounter(HugeType type); - public default Set writeSnapshot(String snapshotDir) { + public default Set createSnapshot(String snapshotDir) { throw new UnsupportedOperationException("writeSnapshot"); } - public default Set readSnapshot(String snapshotDir) { + public default Set resumeSnapshot(String snapshotDir) { throw new UnsupportedOperationException("readSnapshot"); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java index e7f815b64e..12058d7b73 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/BackendStoreProvider.java @@ -54,9 +54,9 @@ public interface BackendStoreProvider { public void initSystemInfo(HugeGraph graph); - public void writeSnapshot(); + public void createSnapshot(); - public void readSnapshot(); + public void resumeSnapshot(); public void listen(EventListener listener); diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java index df049cf8dc..2f55018563 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftBackendStoreProvider.java @@ -199,7 +199,7 @@ public void initSystemInfo(HugeGraph graph) { } @Override - public void writeSnapshot() { + public void createSnapshot() { StoreCommand command = new StoreCommand(StoreType.ALL, StoreAction.SNAPSHOT, null); StoreClosure closure = new StoreClosure(command); @@ -208,7 +208,7 @@ public void writeSnapshot() { } @Override - public void readSnapshot() { + public void resumeSnapshot() { // jraft doesn't expose API to load snapshot throw new UnsupportedOperationException(); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java index 5b2815fd08..e4c7f44385 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/RaftSharedContext.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Paths; +import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; @@ -43,6 +44,7 @@ import com.baidu.hugegraph.HugeException; import com.baidu.hugegraph.HugeGraphParams; import com.baidu.hugegraph.backend.cache.Cache; +import com.baidu.hugegraph.backend.id.Id; import com.baidu.hugegraph.backend.store.BackendStore; import com.baidu.hugegraph.backend.store.raft.rpc.ListPeersProcessor; import com.baidu.hugegraph.backend.store.raft.rpc.RaftRequests.StoreType; @@ -259,7 +261,7 @@ public void clearCache() { this.notifyCache(Cache.ACTION_CLEAR, HugeType.VERTEX, null); } - protected void notifyCache(String action, HugeType type, Object id) { + protected void notifyCache(String action, HugeType type, List ids) { EventHub eventHub; if (type.isGraph()) { eventHub = this.params.graphEventHub(); @@ -270,7 +272,15 @@ protected void notifyCache(String action, HugeType type, Object id) { } try { // How to avoid update cache from server info - eventHub.notify(Events.CACHE, action, type, id); + if (ids == null) { + eventHub.call(Events.CACHE, action, type); + } else { + if (ids.size() == 1) { + eventHub.call(Events.CACHE, action, type, ids.get(0)); + } else { + eventHub.call(Events.CACHE, action, type, ids.toArray()); + } + } } catch (RejectedExecutionException e) { LOG.warn("Can't update cache due to EventHub is too busy"); } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java index f8ac2fe47a..96e8833c0c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreSnapshotFile.java @@ -64,12 +64,15 @@ public void save(SnapshotWriter writer, Closure done, // Write snapshot to real directory Set snapshotDirs = this.doSnapshotSave(); executor.execute(() -> { + long begin = System.currentTimeMillis(); Set tarSnapshotFiles = this.compressSnapshotDir(snapshotDirs, done); String jraftSnapshotPath = this.writeManifest(writer, tarSnapshotFiles, done); this.deleteSnapshotDir(snapshotDirs, done); this.compressJraftSnapshotDir(writer, jraftSnapshotPath, done); + LOG.info("Compress snapshot cost {}ms", + System.currentTimeMillis() - begin); }); } catch (Throwable t) { LOG.error("Failed to save snapshot", t); @@ -111,7 +114,7 @@ public boolean load(SnapshotReader reader) { private Set doSnapshotSave() { Set snapshotDirs = new HashSet<>(); for (RaftBackendStore store : this.stores) { - snapshotDirs.addAll(store.originStore().writeSnapshot(SNAPSHOT_DIR)); + snapshotDirs.addAll(store.originStore().createSnapshot(SNAPSHOT_DIR)); } LOG.info("All snapshot dirs: {}", snapshotDirs); return snapshotDirs; @@ -119,7 +122,7 @@ private Set doSnapshotSave() { private void doSnapshotLoad() { for (RaftBackendStore store : this.stores) { - store.originStore().readSnapshot(SNAPSHOT_DIR); + store.originStore().resumeSnapshot(SNAPSHOT_DIR); } } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java index 12efc35682..2b8981015c 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/backend/store/raft/StoreStateMachine.java @@ -40,7 +40,6 @@ import com.baidu.hugegraph.backend.query.Query; import com.baidu.hugegraph.backend.serializer.BytesBuffer; import com.baidu.hugegraph.backend.store.BackendAction; -import com.baidu.hugegraph.backend.store.BackendEntry; import com.baidu.hugegraph.backend.store.BackendMutation; import com.baidu.hugegraph.backend.store.BackendStore; import com.baidu.hugegraph.backend.store.raft.RaftBackendStore.IncrCounter; @@ -86,21 +85,13 @@ private void updateCacheIfNeeded(BackendMutation mutation, return; } for (HugeType type : mutation.types()) { - if (type.isSchema()) { - java.util.Iterator it = mutation.mutation(type); - while (it.hasNext()) { - BackendEntry entry = it.next().entry(); - this.context.notifyCache(Cache.ACTION_INVALID, type, - entry.originId()); - } - } else if (type.isGraph()) { - List ids = new ArrayList<>((int) Query.COMMIT_BATCH); + List ids = new ArrayList<>((int) Query.COMMIT_BATCH); + if (type.isSchema() || type.isGraph()) { java.util.Iterator it = mutation.mutation(type); while (it.hasNext()) { ids.add(it.next().entry().originId()); } - this.context.notifyCache(Cache.ACTION_INVALID, type, - ids.toArray()); + this.context.notifyCache(Cache.ACTION_INVALID, type, ids); } else { // Ignore other types due to not cached } diff --git a/hugegraph-core/src/main/java/com/baidu/hugegraph/security/HugeSecurityManager.java b/hugegraph-core/src/main/java/com/baidu/hugegraph/security/HugeSecurityManager.java index a97f385bea..8043866cbb 100644 --- a/hugegraph-core/src/main/java/com/baidu/hugegraph/security/HugeSecurityManager.java +++ b/hugegraph-core/src/main/java/com/baidu/hugegraph/security/HugeSecurityManager.java @@ -110,7 +110,8 @@ public class HugeSecurityManager extends SecurityManager { ); private static final Set SOFA_RPC_CLASSES = ImmutableSet.of( - "com.alipay.sofa.rpc.tracer.sofatracer.RpcSofaTracer" + "com.alipay.sofa.rpc.tracer.sofatracer.RpcSofaTracer", + "com.alipay.sofa.rpc.client.AbstractCluster" ); @Override diff --git a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java index c3ed2528c7..f73899bdf9 100644 --- a/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java +++ b/hugegraph-rocksdb/src/main/java/com/baidu/hugegraph/backend/store/rocksdb/RocksDBStore.java @@ -608,7 +608,7 @@ protected Session session(HugeType tableType) { } @Override - public Set writeSnapshot(String snapshotPrefix) { + public Set createSnapshot(String snapshotPrefix) { Lock readLock = this.storeLock.readLock(); readLock.lock(); try { @@ -641,7 +641,7 @@ public Set writeSnapshot(String snapshotPrefix) { } @Override - public Set readSnapshot(String snapshotPrefix) { + public Set resumeSnapshot(String snapshotPrefix) { Lock readLock = this.storeLock.readLock(); readLock.lock(); try { diff --git a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java index 7c070da341..af6609c8db 100644 --- a/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java +++ b/hugegraph-test/src/main/java/com/baidu/hugegraph/core/EdgeCoreTest.java @@ -5489,7 +5489,11 @@ public void testRemoveEdgesOfSuperVertex() { guido.remove(); // Clear all - graph.truncateBackend(); + try { + graph.truncateBackend(); + } catch (UnsupportedOperationException ignored) { + // pass + } } @Test