Support resource files tagged by version to reduce initial data requests

common/src/main/java/bisq/common/app/Version.java

@@ -17,6 +17,9 @@
 
 package bisq.common.app;
 
+import java.util.Arrays;
+import java.util.List;
+
 import lombok.extern.slf4j.Slf4j;
 
 import static com.google.common.base.Preconditions.checkArgument;
@@ -29,6 +32,11 @@ public class Version {
     // We use semantic versioning with major, minor and patch
     public static final String VERSION = "1.3.9";
 
+    /**
+     * Holds a list of the versions of tagged resource files for optimizing the getData requests.
+     */
+    public static final List<String> HISTORY = Arrays.asList("1.4.0");
+
     public static int getMajorVersion(String version) {
         return getSubVersion(version, 0);
     }
@@ -45,7 +53,7 @@ public static boolean isNewVersion(String newVersion) {
         return isNewVersion(newVersion, VERSION);
     }
 
-    static boolean isNewVersion(String newVersion, String currentVersion) {
+    public static boolean isNewVersion(String newVersion, String currentVersion) {
         if (newVersion.equals(currentVersion))
             return false;
         else if (getMajorVersion(newVersion) > getMajorVersion(currentVersion))

common/src/main/java/bisq/common/storage/Storage.java

@@ -77,12 +77,17 @@ public Storage(@Named(Config.STORAGE_DIR) File dir,
         this.corruptedDatabaseFilesHandler = corruptedDatabaseFilesHandler;
     }
 
+    @Nullable
+    public T getPersisted(String fileName) {
+        return getPersisted(new File(dir, fileName));
+    }
+
     @Nullable
     public T initAndGetPersistedWithFileName(String fileName, long delay) {
         this.fileName = fileName;
         storageFile = new File(dir, fileName);
         fileManager = new FileManager<>(dir, storageFile, delay, persistenceProtoResolver);
-        return getPersisted();
+        return getPersisted(storageFile);
     }
 
     @Nullable
@@ -96,7 +101,7 @@ public T initAndGetPersisted(T persistable, String fileName, long delay) {
         this.fileName = fileName;
         storageFile = new File(dir, fileName);
         fileManager = new FileManager<>(dir, storageFile, delay, persistenceProtoResolver);
-        return getPersisted();
+        return getPersisted(storageFile);
     }
 
     public void queueUpForSave() {
@@ -144,7 +149,7 @@ public void remove(String fileName) {
     // We do the file read on the UI thread to avoid problems from multi threading.
     // Data are small and read is done only at startup, so it is no performance issue.
     @Nullable
-    private T getPersisted() {
+    private T getPersisted(File storageFile) {
         if (storageFile.exists()) {
             long now = System.currentTimeMillis();
             try {

core/src/main/java/bisq/core/account/sign/SignedWitnessStorageService.java

@@ -24,8 +24,8 @@
 import bisq.common.config.Config;
 import bisq.common.storage.Storage;
 
-import javax.inject.Named;
 import javax.inject.Inject;
+import javax.inject.Named;
 
 import java.io.File;
 

core/src/main/java/bisq/core/account/sign/SignedWitnessStore.java

@@ -19,18 +19,13 @@
 
 
 import bisq.network.p2p.storage.P2PDataStorage;
-import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-
-import bisq.common.proto.persistable.ThreadedPersistableEnvelope;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
 
 import com.google.protobuf.Message;
 
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 
@@ -40,9 +35,7 @@
  * definition and provide a hashMap for the domain access.
  */
 @Slf4j
-public class SignedWitnessStore implements ThreadedPersistableEnvelope {
-    @Getter
-    private Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
+public class SignedWitnessStore extends PersistableNetworkPayloadStore {
 
     SignedWitnessStore() {
     }

core/src/main/java/bisq/core/account/witness/AccountAgeWitnessStorageService.java

@@ -24,8 +24,8 @@
 import bisq.common.config.Config;
 import bisq.common.storage.Storage;
 
-import javax.inject.Named;
 import javax.inject.Inject;
+import javax.inject.Named;
 
 import java.io.File;
 

core/src/main/java/bisq/core/account/witness/AccountAgeWitnessStore.java

@@ -18,18 +18,13 @@
 package bisq.core.account.witness;
 
 import bisq.network.p2p.storage.P2PDataStorage;
-import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-
-import bisq.common.proto.persistable.ThreadedPersistableEnvelope;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
 
 import com.google.protobuf.Message;
 
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 
@@ -39,9 +34,7 @@
  * definition and provide a hashMap for the domain access.
  */
 @Slf4j
-public class AccountAgeWitnessStore implements ThreadedPersistableEnvelope {
-    @Getter
-    private Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
+public class AccountAgeWitnessStore extends PersistableNetworkPayloadStore {
 
     AccountAgeWitnessStore() {
     }

core/src/main/java/bisq/core/dao/governance/blindvote/storage/BlindVoteStore.java

@@ -18,18 +18,13 @@
 package bisq.core.dao.governance.blindvote.storage;
 
 import bisq.network.p2p.storage.P2PDataStorage;
-import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-
-import bisq.common.proto.persistable.ThreadedPersistableEnvelope;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
 
 import com.google.protobuf.Message;
 
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 
@@ -39,9 +34,7 @@
  * definition and provide a hashMap for the domain access.
  */
 @Slf4j
-public class BlindVoteStore implements ThreadedPersistableEnvelope {
-    @Getter
-    private Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
+public class BlindVoteStore extends PersistableNetworkPayloadStore {
 
     BlindVoteStore() {
     }

core/src/main/java/bisq/core/dao/governance/proposal/ProposalService.java

@@ -216,6 +216,7 @@ private void fillListFromProtectedStore() {
         p2PService.getDataMap().values().forEach(e -> onProtectedDataAdded(e, false));
     }
 
+    //todo use proposalStorageService
     private void fillListFromAppendOnlyDataStore() {
         p2PService.getP2PDataStorage().getAppendOnlyDataStoreMap().values().forEach(e -> onAppendOnlyDataAdded(e, false));
     }

core/src/main/java/bisq/core/dao/governance/proposal/storage/appendonly/ProposalStore.java

@@ -18,18 +18,13 @@
 package bisq.core.dao.governance.proposal.storage.appendonly;
 
 import bisq.network.p2p.storage.P2PDataStorage;
-import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-
-import bisq.common.proto.persistable.ThreadedPersistableEnvelope;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
 
 import com.google.protobuf.Message;
 
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 
@@ -39,9 +34,7 @@
  * definition and provide a hashMap for the domain access.
  */
 @Slf4j
-public class ProposalStore implements ThreadedPersistableEnvelope {
-    @Getter
-    private Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
+public class ProposalStore extends PersistableNetworkPayloadStore {
 
     ProposalStore() {
     }

core/src/main/java/bisq/core/provider/ProvidersRepository.java

@@ -79,11 +79,12 @@ public void applyBannedNodes(@Nullable List<String> bannedNodes) {
         fillProviderList();
         selectNextProviderBaseUrl();
 
-        if (bannedNodes == null)
+        if (bannedNodes == null) {
             log.info("Selected provider baseUrl={}, providerList={}", baseUrl, providerList);
-        else
+        } else if (!bannedNodes.isEmpty()) {
             log.warn("We have banned provider nodes: bannedNodes={}, selected provider baseUrl={}, providerList={}",
                     bannedNodes, baseUrl, providerList);
+        }
     }
 
     public void selectNextProviderBaseUrl() {

core/src/main/java/bisq/core/support/dispute/agent/DisputeAgentService.java

@@ -100,8 +100,11 @@ public Map<NodeAddress, T> getDisputeAgents() {
         } else {
             bannedDisputeAgents = null;
         }
-        if (bannedDisputeAgents != null)
+
+        if (bannedDisputeAgents != null && !bannedDisputeAgents.isEmpty()) {
             log.warn("bannedDisputeAgents=" + bannedDisputeAgents);
+        }
+
         Set<T> disputeAgentSet = getDisputeAgentSet(bannedDisputeAgents);
 
         Map<NodeAddress, T> map = new HashMap<>();

core/src/main/java/bisq/core/trade/statistics/TradeStatistics2StorageService.java

@@ -17,25 +17,21 @@
 
 package bisq.core.trade.statistics;
 
-import bisq.network.p2p.storage.P2PDataStorage;
 import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-import bisq.network.p2p.storage.persistence.MapStoreService;
+import bisq.network.p2p.storage.persistence.HistoricalDataStoreService;
 
 import bisq.common.config.Config;
 import bisq.common.storage.Storage;
 
-import javax.inject.Named;
-
 import javax.inject.Inject;
+import javax.inject.Named;
 
 import java.io.File;
 
-import java.util.Map;
-
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-public class TradeStatistics2StorageService extends MapStoreService<TradeStatistics2Store, PersistableNetworkPayload> {
+public class TradeStatistics2StorageService extends HistoricalDataStoreService<TradeStatistics2Store> {
     private static final String FILE_NAME = "TradeStatistics2Store";
 
 
@@ -59,11 +55,6 @@ public String getFileName() {
         return FILE_NAME;
     }
 
-    @Override
-    public Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> getMap() {
-        return store.getMap();
-    }
-
     @Override
     public boolean canHandle(PersistableNetworkPayload payload) {
         return payload instanceof TradeStatistics2;
@@ -78,9 +69,4 @@ public boolean canHandle(PersistableNetworkPayload payload) {
     protected TradeStatistics2Store createStore() {
         return new TradeStatistics2Store();
     }
-
-    @Override
-    protected void readStore() {
-        super.readStore();
-    }
 }

core/src/main/java/bisq/core/trade/statistics/TradeStatistics2Store.java

@@ -18,18 +18,13 @@
 package bisq.core.trade.statistics;
 
 import bisq.network.p2p.storage.P2PDataStorage;
-import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
-
-import bisq.common.proto.persistable.ThreadedPersistableEnvelope;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
 
 import com.google.protobuf.Message;
 
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -38,9 +33,7 @@
  * definition and provide a hashMap for the domain access.
  */
 @Slf4j
-public class TradeStatistics2Store implements ThreadedPersistableEnvelope {
-    @Getter
-    private final Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
+public class TradeStatistics2Store extends PersistableNetworkPayloadStore {
 
     TradeStatistics2Store() {
     }

core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java

@@ -79,15 +79,15 @@ public void onAllServicesInitialized() {
                 addToSet((TradeStatistics2) payload);
         });
 
-        Set<TradeStatistics2> collect = tradeStatistics2StorageService.getMap().values().stream()
+        Set<TradeStatistics2> set = tradeStatistics2StorageService.getMapOfAllData().values().stream()
                 .filter(e -> e instanceof TradeStatistics2)
                 .map(e -> (TradeStatistics2) e)
                 .map(WrapperTradeStatistics2::new)
                 .distinct()
                 .map(WrapperTradeStatistics2::unwrap)
                 .filter(TradeStatistics2::isValid)
                 .collect(Collectors.toSet());
-        observableTradeStatisticsSet.addAll(collect);
+        observableTradeStatisticsSet.addAll(set);
 
         priceFeedService.applyLatestBisqMarketPrice(observableTradeStatisticsSet);
 
@@ -99,7 +99,6 @@ public ObservableSet<TradeStatistics2> getObservableTradeStatisticsSet() {
     }
 
     private void addToSet(TradeStatistics2 tradeStatistics) {
-
         if (!observableTradeStatisticsSet.contains(tradeStatistics)) {
             Optional<TradeStatistics2> duplicate = observableTradeStatisticsSet.stream().filter(
                     e -> e.getOfferId().equals(tradeStatistics.getOfferId())).findAny();

p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java

@@ -88,22 +88,22 @@ public void handle(GetDataRequest getDataRequest, final Connection connection) {
                 .map(e -> "node address " + e.getFullAddress())
                 .orElseGet(() -> "connection UID " + connection.getUid());
 
-        AtomicBoolean outPersistableNetworkPayloadOutputTruncated = new AtomicBoolean(false);
-        AtomicBoolean outProtectedStoragePayloadOutputTruncated = new AtomicBoolean(false);
+        AtomicBoolean wasPersistableNetworkPayloadsTruncated = new AtomicBoolean(false);
+        AtomicBoolean wasProtectedStorageEntriesTruncated = new AtomicBoolean(false);
         GetDataResponse getDataResponse = dataStorage.buildGetDataResponse(
                 getDataRequest,
                 MAX_ENTRIES,
-                outPersistableNetworkPayloadOutputTruncated,
-                outProtectedStoragePayloadOutputTruncated,
+                wasPersistableNetworkPayloadsTruncated,
+                wasProtectedStorageEntriesTruncated,
                 connection.getCapabilities());
 
-        if (outPersistableNetworkPayloadOutputTruncated.get()) {
+        if (wasPersistableNetworkPayloadsTruncated.get()) {
             log.warn("The getData request from peer with {} caused too much PersistableNetworkPayload " +
                             "entries to get delivered. We limited the entries for the response to {} entries",
                     connectionInfo, MAX_ENTRIES);
         }
 
-        if (outProtectedStoragePayloadOutputTruncated.get()) {
+        if (wasProtectedStorageEntriesTruncated.get()) {
             log.warn("The getData request from peer with {} caused too much ProtectedStorageEntry " +
                             "entries to get delivered. We limited the entries for the response to {} entries",
                     connectionInfo, MAX_ENTRIES);