Reduce initial data request

common/src/main/java/bisq/common/app/Version.java

@@ -17,6 +17,9 @@
 
 package bisq.common.app;
 
+import java.util.Arrays;
+import java.util.List;
+
 import lombok.extern.slf4j.Slf4j;
 
 import static com.google.common.base.Preconditions.checkArgument;
@@ -29,6 +32,11 @@ public class Version {
     // We use semantic versioning with major, minor and patch
     public static final String VERSION = "1.3.9";
 
+    /**
+     * Holds a list of the versions of tagged resource files for optimizing the getData requests.
+     */
+    public static final List<String> HISTORY = Arrays.asList("1.4.0");
+
     public static int getMajorVersion(String version) {
         return getSubVersion(version, 0);
     }
@@ -45,7 +53,7 @@ public static boolean isNewVersion(String newVersion) {
         return isNewVersion(newVersion, VERSION);
     }
 
-    static boolean isNewVersion(String newVersion, String currentVersion) {
+    public static boolean isNewVersion(String newVersion, String currentVersion) {
         if (newVersion.equals(currentVersion))
             return false;
         else if (getMajorVersion(newVersion) > getMajorVersion(currentVersion))

common/src/main/java/bisq/common/storage/Storage.java

@@ -77,12 +77,17 @@ public Storage(@Named(Config.STORAGE_DIR) File dir,
         this.corruptedDatabaseFilesHandler = corruptedDatabaseFilesHandler;
     }
 
+    @Nullable
+    public T getPersisted(String fileName) {
+        return getPersisted(new File(dir, fileName));
+    }
+
     @Nullable
     public T initAndGetPersistedWithFileName(String fileName, long delay) {
         this.fileName = fileName;
         storageFile = new File(dir, fileName);
         fileManager = new FileManager<>(dir, storageFile, delay, persistenceProtoResolver);
-        return getPersisted();
+        return getPersisted(storageFile);
     }
 
     @Nullable
@@ -96,7 +101,7 @@ public T initAndGetPersisted(T persistable, String fileName, long delay) {
         this.fileName = fileName;
         storageFile = new File(dir, fileName);
         fileManager = new FileManager<>(dir, storageFile, delay, persistenceProtoResolver);
-        return getPersisted();
+        return getPersisted(storageFile);
     }
 
     public void queueUpForSave() {
@@ -144,7 +149,7 @@ public void remove(String fileName) {
     // We do the file read on the UI thread to avoid problems from multi threading.
     // Data are small and read is done only at startup, so it is no performance issue.
     @Nullable
-    private T getPersisted() {
+    private T getPersisted(File storageFile) {
         if (storageFile.exists()) {
             long now = System.currentTimeMillis();
             try {

core/src/main/java/bisq/core/account/sign/SignedWitnessStore.java

@@ -19,18 +19,13 @@
 
 
 import bisq.network.p2p.storage.P2PDataStorage;
-import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-
-import bisq.common.proto.persistable.ThreadedPersistableEnvelope;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
 
 import com.google.protobuf.Message;
 
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 
@@ -40,9 +35,7 @@
  * definition and provide a hashMap for the domain access.
  */
 @Slf4j
-public class SignedWitnessStore implements ThreadedPersistableEnvelope {
-    @Getter
-    private Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
+public class SignedWitnessStore extends PersistableNetworkPayloadStore {
 
     SignedWitnessStore() {
     }

core/src/main/java/bisq/core/account/witness/AccountAgeWitnessStore.java

@@ -18,18 +18,13 @@
 package bisq.core.account.witness;
 
 import bisq.network.p2p.storage.P2PDataStorage;
-import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-
-import bisq.common.proto.persistable.ThreadedPersistableEnvelope;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
 
 import com.google.protobuf.Message;
 
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 
@@ -39,9 +34,7 @@
  * definition and provide a hashMap for the domain access.
  */
 @Slf4j
-public class AccountAgeWitnessStore implements ThreadedPersistableEnvelope {
-    @Getter
-    private Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
+public class AccountAgeWitnessStore extends PersistableNetworkPayloadStore {
 
     AccountAgeWitnessStore() {
     }

core/src/main/java/bisq/core/dao/governance/blindvote/storage/BlindVoteStore.java

@@ -18,18 +18,13 @@
 package bisq.core.dao.governance.blindvote.storage;
 
 import bisq.network.p2p.storage.P2PDataStorage;
-import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-
-import bisq.common.proto.persistable.ThreadedPersistableEnvelope;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
 
 import com.google.protobuf.Message;
 
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 
@@ -39,9 +34,7 @@
  * definition and provide a hashMap for the domain access.
  */
 @Slf4j
-public class BlindVoteStore implements ThreadedPersistableEnvelope {
-    @Getter
-    private Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
+public class BlindVoteStore extends PersistableNetworkPayloadStore {
 
     BlindVoteStore() {
     }

core/src/main/java/bisq/core/dao/governance/proposal/storage/appendonly/ProposalStore.java

@@ -18,18 +18,13 @@
 package bisq.core.dao.governance.proposal.storage.appendonly;
 
 import bisq.network.p2p.storage.P2PDataStorage;
-import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-
-import bisq.common.proto.persistable.ThreadedPersistableEnvelope;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
 
 import com.google.protobuf.Message;
 
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 
@@ -39,9 +34,7 @@
  * definition and provide a hashMap for the domain access.
  */
 @Slf4j
-public class ProposalStore implements ThreadedPersistableEnvelope {
-    @Getter
-    private Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
+public class ProposalStore extends PersistableNetworkPayloadStore {
 
     ProposalStore() {
     }

core/src/main/java/bisq/core/provider/ProvidersRepository.java

@@ -79,11 +79,12 @@ public void applyBannedNodes(@Nullable List<String> bannedNodes) {
         fillProviderList();
         selectNextProviderBaseUrl();
 
-        if (bannedNodes == null)
+        if (bannedNodes == null) {
             log.info("Selected provider baseUrl={}, providerList={}", baseUrl, providerList);
-        else
+        } else if (!bannedNodes.isEmpty()) {
             log.warn("We have banned provider nodes: bannedNodes={}, selected provider baseUrl={}, providerList={}",
                     bannedNodes, baseUrl, providerList);
+        }
     }
 
     public void selectNextProviderBaseUrl() {

core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentService.java

@@ -100,8 +100,11 @@ public Map<NodeAddress, T> getDisputeAgents() {
         } else {
             bannedDisputeAgents = null;
         }
-        if (bannedDisputeAgents != null)
+
+        if (bannedDisputeAgents != null && !bannedDisputeAgents.isEmpty()) {
             log.warn("bannedDisputeAgents=" + bannedDisputeAgents);
+        }
+
         Set<T> disputeAgentSet = getDisputeAgentSet(bannedDisputeAgents);
 
         Map<NodeAddress, T> map = new HashMap<>();

core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java

@@ -24,9 +24,8 @@
 import bisq.common.config.Config;
 import bisq.common.storage.Storage;
 
-import javax.inject.Named;
-
 import javax.inject.Inject;
+import javax.inject.Named;
 
 import java.io.File;
 
@@ -64,6 +63,10 @@ public Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> getMap() {
         return store.getMap();
     }
 
+    public Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> getMapOfAllData() {
+        return getMap();
+    }
+
     @Override
     public boolean canHandle(PersistableNetworkPayload payload) {
         return payload instanceof TradeStatistics2;
@@ -78,9 +81,4 @@ public boolean canHandle(PersistableNetworkPayload payload) {
     protected TradeStatistics2Store createStore() {
         return new TradeStatistics2Store();
     }
-
-    @Override
-    protected void readStore() {
-        super.readStore();
-    }
 }

core/src/main/java/bisq/core/trade/statistics/TradeStatistics2Store.java

@@ -18,18 +18,13 @@
 package bisq.core.trade.statistics;
 
 import bisq.network.p2p.storage.P2PDataStorage;
-import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-
-import bisq.common.proto.persistable.ThreadedPersistableEnvelope;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
 
 import com.google.protobuf.Message;
 
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -38,9 +33,7 @@
  * definition and provide a hashMap for the domain access.
  */
 @Slf4j
-public class TradeStatistics2Store implements ThreadedPersistableEnvelope {
-    @Getter
-    private final Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
+public class TradeStatistics2Store extends PersistableNetworkPayloadStore {
 
     TradeStatistics2Store() {
     }

core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java

@@ -79,15 +79,15 @@ public void onAllServicesInitialized() {
                 addToSet((TradeStatistics2) payload);
         });
 
-        Set<TradeStatistics2> collect = tradeStatistics2StorageService.getMap().values().stream()
+        Set<TradeStatistics2> set = tradeStatistics2StorageService.getMapOfAllData().values().stream()
                 .filter(e -> e instanceof TradeStatistics2)
                 .map(e -> (TradeStatistics2) e)
                 .map(WrapperTradeStatistics2::new)
                 .distinct()
                 .map(WrapperTradeStatistics2::unwrap)
                 .filter(TradeStatistics2::isValid)
                 .collect(Collectors.toSet());
-        observableTradeStatisticsSet.addAll(collect);
+        observableTradeStatisticsSet.addAll(set);
 
         priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet);
 
@@ -99,7 +99,6 @@ public ObservableSet<TradeStatistics2> getObservableTradeStatisticsSet() {
     }
 
     private void addToSet(TradeStatistics2 tradeStatistics) {
-
         if (!observableTradeStatisticsSet.contains(tradeStatistics)) {
             Optional<TradeStatistics2> duplicate = observableTradeStatisticsSet.stream().filter(
                     e -> e.getOfferId().equals(tradeStatistics.getOfferId())).findAny();

p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java

@@ -88,22 +88,22 @@ public void handle(GetDataRequest getDataRequest, final Connection connection) {
                 .map(e -> "node address " + e.getFullAddress())
                 .orElseGet(() -> "connection UID " + connection.getUid());
 
-        AtomicBoolean outPersistableNetworkPayloadOutputTruncated = new AtomicBoolean(false);
-        AtomicBoolean outProtectedStoragePayloadOutputTruncated = new AtomicBoolean(false);
+        AtomicBoolean wasPersistableNetworkPayloadsTruncated = new AtomicBoolean(false);
+        AtomicBoolean wasProtectedStorageEntriesTruncated = new AtomicBoolean(false);
         GetDataResponse getDataResponse = dataStorage.buildGetDataResponse(
                 getDataRequest,
                 MAX_ENTRIES,
-                outPersistableNetworkPayloadOutputTruncated,
-                outProtectedStoragePayloadOutputTruncated,
+                wasPersistableNetworkPayloadsTruncated,
+                wasProtectedStorageEntriesTruncated,
                 connection.getCapabilities());
 
-        if (outPersistableNetworkPayloadOutputTruncated.get()) {
+        if (wasPersistableNetworkPayloadsTruncated.get()) {
             log.warn("The getData request from peer with {} caused too much PersistableNetworkPayload " +
                             "entries to get delivered. We limited the entries for the response to {} entries",
                     connectionInfo, MAX_ENTRIES);
         }
 
-        if (outProtectedStoragePayloadOutputTruncated.get()) {
+        if (wasProtectedStorageEntriesTruncated.get()) {
             log.warn("The getData request from peer with {} caused too much ProtectedStorageEntry " +
                             "entries to get delivered. We limited the entries for the response to {} entries",
                     connectionInfo, MAX_ENTRIES);

p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataHandler.java

@@ -260,15 +260,16 @@ private void logContents(NetworkEnvelope networkEnvelope,
 
         // Log different data types
         StringBuilder sb = new StringBuilder();
-        sb.append("\n#################################################################\n");
-        sb.append("Connected to node: " + peersNodeAddress.getFullAddress() + "\n");
+        String sep = System.lineSeparator();
+        sb.append(sep).append("#################################################################").append(sep);
+        sb.append("Connected to node: ").append(peersNodeAddress.getFullAddress()).append(sep);
         int items = dataSet.size() + persistableNetworkPayloadSet.size();
         sb.append("Received ").append(items).append(" instances from a ")
-                .append(getDataRequestType).append("\n");
+                .append(getDataRequestType).append(sep);
         payloadByClassName.forEach((key, value) -> sb.append(key)
                 .append(": ")
                 .append(value.size())
-                .append("\n"));
+                .append(sep));
         sb.append("#################################################################");
         log.info(sb.toString());
     }

p2p/src/main/java/bisq/network/p2p/peers/getdata/messages/GetDataRequest.java

@@ -27,6 +27,8 @@
 import lombok.Getter;
 import lombok.ToString;
 
+import javax.annotation.Nullable;
+
 @EqualsAndHashCode(callSuper = true)
 @Getter
 @ToString
@@ -35,11 +37,18 @@ public abstract class GetDataRequest extends NetworkEnvelope implements Extended
     // Keys for ProtectedStorageEntry items to be excluded from the request because the peer has them already
     protected final Set<byte[]> excludedKeys;
 
+    // Added at v1.4.0
+    // The version of the requester. Used for response to send potentially missing historical data
+    @Nullable
+    protected final String version;
+
     public GetDataRequest(int messageVersion,
                           int nonce,
-                          Set<byte[]> excludedKeys) {
+                          Set<byte[]> excludedKeys,
+                          @Nullable String version) {
         super(messageVersion);
         this.nonce = nonce;
         this.excludedKeys = excludedKeys;
+        this.version = version;
     }
 }