From c9ea76cdb90bd9fc63631607881ff8b17f073efe Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Fri, 14 Feb 2020 21:43:02 -0800 Subject: [PATCH 01/19] Add persistent node setup --- build.gradle | 1 + gradle/check-licenses.gradle | 2 +- gradle/versions.gradle | 2 +- .../acceptance/send/SingleNodeSendTest.java | 2 - .../receive/DualNodesSendReceiveTest.java | 12 +- .../DualNodesPrivacyGroupsTest.java | 10 +- ...NodesSendReceiveUsingPrivacyGroupTest.java | 12 +- .../java/net/consensys/orion/cmd/Orion.java | 89 +++++--- .../net/consensys/orion/config/Config.java | 19 +- .../serialization/PublicKeyURISerializer.java | 39 ++++ .../http/handler/knownnodes/KnownNode.java | 20 +- .../handler/knownnodes/KnownNodesHandler.java | 12 +- .../handler/partyinfo/PartyInfoHandler.java | 11 +- .../privacy/CreatePrivacyGroupHandler.java | 12 +- .../privacy/DeletePrivacyGroupHandler.java | 12 +- .../orion/network/ConcurrentNetworkNodes.java | 154 -------------- .../orion/network/NetworkDiscovery.java | 36 ++-- .../consensys/orion/network/NetworkNodes.java | 28 ++- .../orion/network/PersistentNetworkNodes.java | 191 ++++++++++++++++++ .../orion/network/ReadOnlyNetworkNodes.java | 81 ++++++++ .../payload/DistributePayloadManager.java | 25 ++- .../storage/EncryptedPayloadStorage.java | 4 +- .../orion/storage/OrionSQLKeyValueStore.java | 88 -------- .../orion/storage/PrivacyGroupStorage.java | 4 +- .../storage/QueryPrivacyGroupStorage.java | 4 +- .../net/consensys/orion/helpers/FakePeer.java | 6 +- .../CreatePrivacyGroupHandlerTest.java | 18 +- .../DeletePrivacyGroupHandlerTest.java | 2 +- .../handler/FindPrivacyGroupHandlerTest.java | 2 +- .../orion/http/handler/HandlerTest.java | 16 +- .../http/handler/PartyInfoHandlerTest.java | 38 ++-- .../RetrievePrivacyGroupHandlerTest.java | 2 +- .../orion/http/handler/SendHandlerTest.java | 2 +- .../http/handler/SendRawHandlerTest.java | 4 +- .../handler/SendRawHandlerWithNodeKeys.java | 2 +- .../knownnodes/KnownNodesHandlerTest.java | 15 +- .../orion/network/CaOrTofuNodeClientTest.java | 5 +- .../CertificateAuthorityNodeClientTest.java | 5 +- .../network/ConcurrentNetworkNodesTest.java | 42 ---- .../orion/network/InsecureNodeClientTest.java | 5 +- .../orion/network/NetworkDiscoveryTest.java | 52 +++-- .../orion/network/TofuNodeClientTest.java | 5 +- .../network/WhitelistNodeClientTest.java | 6 +- .../payload/DistributePayloadManagerTest.java | 22 +- .../storage/EncryptedPayloadStorageTest.java | 2 +- .../storage/PrivacyGroupStorageTest.java | 26 ++- 46 files changed, 631 insertions(+), 516 deletions(-) create mode 100644 src/main/java/net/consensys/orion/enclave/sodium/serialization/PublicKeyURISerializer.java delete mode 100644 src/main/java/net/consensys/orion/network/ConcurrentNetworkNodes.java create mode 100644 src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java create mode 100644 src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java delete mode 100644 src/main/java/net/consensys/orion/storage/OrionSQLKeyValueStore.java delete mode 100644 src/test/java/net/consensys/orion/network/ConcurrentNetworkNodesTest.java diff --git a/build.gradle b/build.gradle index 75b89674..9ed4f2a7 100644 --- a/build.gradle +++ b/build.gradle @@ -243,6 +243,7 @@ repositories { jcenter() maven { url "https://repo1.maven.org/maven2/" } maven { url "https://consensys.bintray.com/consensys" } + maven { url "https://repository.apache.org/content/repositories/snapshots/" } } dependencies { diff --git a/gradle/check-licenses.gradle b/gradle/check-licenses.gradle index e03df739..1f1bc23a 100644 --- a/gradle/check-licenses.gradle +++ b/gradle/check-licenses.gradle @@ -130,7 +130,7 @@ downloadLicenses { 'Eclipse Distribution License - v 1.0', ], (epl1): [ - 'Eclipse Public License - v 1.0', + 'Eclipse Public License - v 1.0' ], (epl2): [ 'Eclipse Public License v2.0', diff --git a/gradle/versions.gradle b/gradle/versions.gradle index b9432571..75c465e1 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -53,7 +53,7 @@ dependencyManagement { dependency 'org.apache.tuweni:tuweni-crypto:0.10.0' dependency 'org.apache.tuweni:tuweni-io:0.10.0' dependency 'org.apache.tuweni:tuweni-junit:0.10.0' - dependency 'org.apache.tuweni:tuweni-kv:0.10.0' + dependency 'org.apache.tuweni:tuweni-kv:1.0.0-SNAPSHOT' dependency 'org.apache.tuweni:tuweni-net:0.10.0' dependency 'org.apache.tuweni:tuweni-rlp:0.10.0' diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java index 277ea570..bd4961d0 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java @@ -25,7 +25,6 @@ import net.consensys.orion.exception.OrionErrorCode; import java.net.MalformedURLException; -import java.net.URL; import java.nio.file.Path; import io.vertx.core.Vertx; @@ -84,7 +83,6 @@ static void setUpSingleNode(@TempDirectory final Path tempDir) throws Exception void setUp() throws MalformedURLException { orionLauncher = NodeUtils.startOrion(config); vertx = vertx(); - orionLauncher.addPeer(new URL(NodeUtils.urlString(HOST_NAME, orionLauncher.nodePort()))); httpClient = vertx.createHttpClient(); } diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java index ba04aea0..02342363 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java @@ -29,7 +29,7 @@ import net.consensys.orion.cmd.Orion; import net.consensys.orion.config.Config; import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.network.ConcurrentNetworkNodes; +import net.consensys.orion.network.PersistentNetworkNodes; import net.consensys.orion.utils.Serializer; import java.io.IOException; @@ -69,7 +69,7 @@ class DualNodesSendReceiveTest { private Config firstNodeConfig; private Config secondNodeConfig; - private ConcurrentNetworkNodes networkNodes; + private PersistentNetworkNodes networkNodes; private Orion firstOrionLauncher; private Orion secondOrionLauncher; @@ -129,7 +129,7 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { final Box.PublicKey pk3 = Box.PublicKey.fromBytes(decodeBytes(pubKeyStrings.get(2))); final Box.PublicKey pk4 = Box.PublicKey.fromBytes(decodeBytes(pubKeyStrings.get(3))); final Box.PublicKey pk5 = Box.PublicKey.fromBytes(decodeBytes(pubKeyStrings.get(4))); - networkNodes = new ConcurrentNetworkNodes(NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); + networkNodes = new PersistentNetworkNodes(NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); networkNodes.addNode(Arrays.asList(pk1, pk2), NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); networkNodes.addNode(Arrays.asList(pk3, pk4, pk5), NodeUtils.url("127.0.0.1", secondOrionLauncher.nodePort())); @@ -163,13 +163,13 @@ private String getStringFromResource(final String resourceFileName) { } } - private ConcurrentNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) + private PersistentNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) throws Exception { final Response resp = httpClient.newCall(request).execute(); assertEquals(200, resp.code()); - final ConcurrentNetworkNodes partyInfoResponse = - Serializer.deserialize(HttpContentType.CBOR, ConcurrentNetworkNodes.class, resp.body().bytes()); + final PersistentNetworkNodes partyInfoResponse = + Serializer.deserialize(HttpContentType.CBOR, PersistentNetworkNodes.class, resp.body().bytes()); return partyInfoResponse; } diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java index 4a87ac14..66b5b9bd 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java @@ -35,7 +35,7 @@ import net.consensys.orion.config.Config; import net.consensys.orion.http.handler.privacy.PrivacyGroup; import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.network.ConcurrentNetworkNodes; +import net.consensys.orion.network.PersistentNetworkNodes; import net.consensys.orion.utils.Serializer; import java.nio.file.Path; @@ -124,8 +124,8 @@ void setUpDualNodes(@TempDir final Path tempDir) throws Exception { secondHttpClient = vertx.createHttpClient(); final Box.PublicKey pk1 = Box.PublicKey.fromBytes(decodeBytes(PK_1_B_64)); final Box.PublicKey pk2 = Box.PublicKey.fromBytes(decodeBytes(PK_2_B_64)); - final ConcurrentNetworkNodes networkNodes = - new ConcurrentNetworkNodes(NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); + final PersistentNetworkNodes networkNodes = + new PersistentNetworkNodes(NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); networkNodes.addNode(Collections.singletonList(pk1), NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); networkNodes.addNode(Collections.singletonList(pk2), NodeUtils.url("127.0.0.1", secondOrionLauncher.nodePort())); @@ -142,12 +142,12 @@ void setUpDualNodes(@TempDir final Path tempDir) throws Exception { } - private ConcurrentNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) + private PersistentNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) throws Exception { final Response resp = httpClient.newCall(request).execute(); assertEquals(200, resp.code()); - return Serializer.deserialize(HttpContentType.CBOR, ConcurrentNetworkNodes.class, resp.body().bytes()); + return Serializer.deserialize(HttpContentType.CBOR, PersistentNetworkNodes.class, resp.body().bytes()); } @AfterEach diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java index 3401d52e..51c49af3 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java @@ -32,7 +32,7 @@ import net.consensys.orion.config.Config; import net.consensys.orion.http.handler.receive.ReceiveResponse; import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.network.ConcurrentNetworkNodes; +import net.consensys.orion.network.PersistentNetworkNodes; import net.consensys.orion.utils.Serializer; import java.nio.file.Path; @@ -66,7 +66,7 @@ class DualNodesSendReceiveUsingPrivacyGroupTest { private Config firstNodeConfig; private Config secondNodeConfig; - private ConcurrentNetworkNodes networkNodes; + private PersistentNetworkNodes networkNodes; private Orion firstOrionLauncher; private Orion secondOrionLauncher; @@ -121,7 +121,7 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { secondHttpClient = vertx.createHttpClient(); final Box.PublicKey pk1 = Box.PublicKey.fromBytes(decodeBytes(PK_1_B_64)); final Box.PublicKey pk2 = Box.PublicKey.fromBytes(decodeBytes(PK_2_B_64)); - networkNodes = new ConcurrentNetworkNodes(NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); + networkNodes = new PersistentNetworkNodes(NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); networkNodes.addNode(Collections.singletonList(pk1), NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); networkNodes.addNode(Collections.singletonList(pk2), NodeUtils.url("127.0.0.1", secondOrionLauncher.nodePort())); @@ -138,13 +138,13 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { } - private ConcurrentNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) + private PersistentNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) throws Exception { final Response resp = httpClient.newCall(request).execute(); assertEquals(200, resp.code()); - final ConcurrentNetworkNodes partyInfoResponse = - Serializer.deserialize(HttpContentType.CBOR, ConcurrentNetworkNodes.class, resp.body().bytes()); + final PersistentNetworkNodes partyInfoResponse = + Serializer.deserialize(HttpContentType.CBOR, PersistentNetworkNodes.class, resp.body().bytes()); return partyInfoResponse; } diff --git a/src/main/java/net/consensys/orion/cmd/Orion.java b/src/main/java/net/consensys/orion/cmd/Orion.java index 201a2437..eff897f3 100644 --- a/src/main/java/net/consensys/orion/cmd/Orion.java +++ b/src/main/java/net/consensys/orion/cmd/Orion.java @@ -43,25 +43,26 @@ import net.consensys.orion.http.handler.sendraw.SendRawHandler; import net.consensys.orion.http.handler.upcheck.UpcheckHandler; import net.consensys.orion.http.server.vertx.HttpErrorHandler; -import net.consensys.orion.network.ConcurrentNetworkNodes; import net.consensys.orion.network.NetworkDiscovery; +import net.consensys.orion.network.PersistentNetworkNodes; import net.consensys.orion.payload.DistributePayloadManager; import net.consensys.orion.storage.EncryptedPayloadStorage; import net.consensys.orion.storage.JpaEntityManagerProvider; -import net.consensys.orion.storage.OrionSQLKeyValueStore; import net.consensys.orion.storage.PrivacyGroupStorage; import net.consensys.orion.storage.QueryPrivacyGroupStorage; import net.consensys.orion.storage.Sha512_256StorageKeyBuilder; import net.consensys.orion.storage.Storage; import net.consensys.orion.storage.StorageKeyBuilder; +import net.consensys.orion.storage.Store; import net.consensys.orion.utils.TLS; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -71,6 +72,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; import javax.annotation.Nullable; import io.vertx.core.AsyncResult; @@ -87,10 +89,15 @@ import io.vertx.ext.web.handler.ResponseContentTypeHandler; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.crypto.sodium.Box; import org.apache.tuweni.crypto.sodium.Sodium; +import org.apache.tuweni.io.Base64; +import org.apache.tuweni.kv.EntityManagerKeyValueStore; import org.apache.tuweni.kv.KeyValueStore; import org.apache.tuweni.kv.LevelDBKeyValueStore; import org.apache.tuweni.kv.MapDBKeyValueStore; +import org.apache.tuweni.kv.ProxyKeyValueStore; public class Orion { @@ -106,7 +113,7 @@ public class Orion { } private final Vertx vertx; - private KeyValueStore storage; + private KeyValueStore storage; private NetworkDiscovery discovery; private HttpServer nodeHTTPServer; private HttpServer clientHTTPServer; @@ -130,7 +137,7 @@ public static void main(final String[] args) { public static void configureRoutes( final Vertx vertx, - final ConcurrentNetworkNodes networkNodes, + final PersistentNetworkNodes networkNodes, final Enclave enclave, final Storage storage, final Storage privacyGroupStorage, @@ -171,7 +178,7 @@ public static void configureRoutes( clientRouter.get("/upcheck").produces(TEXT.httpHeaderValue).handler(new UpcheckHandler()); clientRouter.get("/peercount").produces(TEXT.httpHeaderValue).handler( - new PeerCountHandler(() -> networkNodes.nodeURLs().size())); + new PeerCountHandler(() -> networkNodes.nodeURIs().size())); clientRouter.post("/send").produces(JSON.httpHeaderValue).consumes(JSON.httpHeaderValue).handler( new SendHandler(distributePayloadManager)); @@ -331,13 +338,17 @@ public void run(final PrintStream out, final PrintStream err, final Config confi } catch (final IOException ex) { throw new OrionStartException(ex.getMessage(), ex); } - final ConcurrentNetworkNodes networkNodes = new ConcurrentNetworkNodes(config, keyStore.nodeKeys()); - - final Enclave enclave = new SodiumEnclave(keyStore); final Path workDir = config.workDir(); log.info("using working directory {}", workDir); + // create our storage engine + storage = createStorage(config.storage(), workDir); + + final PersistentNetworkNodes networkNodes = new PersistentNetworkNodes(config, keyStore.nodeKeys(), wrap(storage)); + + final Enclave enclave = new SodiumEnclave(keyStore); + try { Files.createDirectories(workDir); } catch (final IOException ex) { @@ -380,9 +391,6 @@ public void run(final PrintStream out, final PrintStream err, final Config confi } } - // create our storage engine - storage = createStorage(config.storage(), workDir); - // Vertx routers final Router nodeRouter = Router.router(vertx).exceptionHandler(log::error); final Router clientRouter = Router.router(vertx).exceptionHandler(log::error); @@ -471,9 +479,9 @@ public void run(final PrintStream out, final PrintStream err, final Config confi CompletableFuture.allOf(nodeFuture, clientFuture).get(); // if there is not a node url in the config, then grab the actual port and use it to set the node url. if (!config.nodeUrl().isPresent()) { - networkNodes.setNodeUrl( - new URL("http", config.nodeNetworkInterface(), nodeHTTPServer.actualPort(), ""), - keyStore.nodeKeys()); + URI nodeURI = + new URI("http", null, config.nodeNetworkInterface(), nodeHTTPServer.actualPort(), null, null, null); + networkNodes.setNodeUrl(nodeURI, keyStore.nodeKeys()); } final CompletableFuture networkDiscoveryFuture = new CompletableFuture<>(); @@ -487,7 +495,7 @@ public void run(final PrintStream out, final PrintStream err, final Config confi } }); CompletableFuture.allOf(networkDiscoveryFuture).get(); - } catch (final ExecutionException | MalformedURLException e) { + } catch (final ExecutionException | URISyntaxException e) { throw new OrionStartException("Orion failed to start: " + e.getCause().getMessage(), e.getCause()); } catch (final InterruptedException e) { throw new OrionStartException("Orion was interrupted while starting services"); @@ -514,7 +522,7 @@ private void writePortsToFile(final Config config, final int nodePort, final int try (final FileOutputStream fileOutputStream = new FileOutputStream(portsFile)) { properties.store( fileOutputStream, - "This file contains the ports used by the running instance of Pantheon. This file will be deleted after the node is shutdown."); + "This file contains the ports used by the running instance of Besu. This file will be deleted after the node is shutdown."); } catch (final Exception e) { log.warn("Error writing ports file", e); } @@ -530,19 +538,34 @@ private Handler> completeFutureInHandler(final Completab }; } - private KeyValueStore createStorage(final String storage, final Path storagePath) { + public static KeyValueStore wrap(KeyValueStore store) { + return ProxyKeyValueStore + .open(store, Box.PublicKey::fromBytes, Box.PublicKey::bytes, Orion::bytesToURI, Orion::uriToBytes); + } + + private static Bytes uriToBytes(URI uri) { + return Bytes.wrap(uri.toString().getBytes(StandardCharsets.UTF_8)); + } + + private static URI bytesToURI(Bytes v) { + try { + return URI.create(new String(v.toArray(), StandardCharsets.UTF_8)); + } catch (IllegalArgumentException e) { + log.warn("Error reading URI", e); + } + return null; + } + + private KeyValueStore createStorage(final String storage, final Path storagePath) { String db = "routerdb"; final String[] storageOptions = storage.split(":", 2); if (storageOptions.length > 1) { db = storageOptions[1]; } - + final Function bytesIdentityFn = Function.identity(); if (storage.toLowerCase().startsWith("mapdb")) { - try { - return MapDBKeyValueStore.open(storagePath.resolve(db)); - } catch (final IOException e) { - throw new OrionStartException("Couldn't create MapDB store: " + db, e); - } + return MapDBKeyValueStore + .open(storagePath.resolve(db), bytesIdentityFn, bytesIdentityFn, bytesIdentityFn, bytesIdentityFn); } else if (storage.toLowerCase().startsWith("leveldb")) { try { return LevelDBKeyValueStore.open(storagePath.resolve(db)); @@ -551,7 +574,17 @@ private KeyValueStore createStorage(final String storage, final Path storagePath } } else if (storage.toLowerCase().startsWith("sql")) { final JpaEntityManagerProvider jpaEntityManagerProvider = new JpaEntityManagerProvider(db); - return new OrionSQLKeyValueStore(jpaEntityManagerProvider); + return ProxyKeyValueStore.open( + EntityManagerKeyValueStore.open(jpaEntityManagerProvider::createEntityManager, Store.class, Store::getKey), + Base64::decode, + Base64::encode, + store -> Bytes.concatenate(Base64.decode(store.getKey()), Bytes.wrap(store.getValue())), + value -> { + Store store = new Store(); + store.setKey(Base64.encode(value.slice(0, 32))); + store.setValue(value.slice(32).toArrayUnsafe()); + return store; + }); } else { throw new OrionStartException("unsupported storage mechanism: " + storage); } @@ -603,8 +636,4 @@ public int nodePort() { public int clientPort() { return clientHTTPServer.actualPort(); } - - public void addPeer(final URL url) { - discovery.addPeer(url); - } } diff --git a/src/main/java/net/consensys/orion/config/Config.java b/src/main/java/net/consensys/orion/config/Config.java index 61306d5a..f2c50895 100644 --- a/src/main/java/net/consensys/orion/config/Config.java +++ b/src/main/java/net/consensys/orion/config/Config.java @@ -245,10 +245,10 @@ public Optional passwords() { * Storage engine used to save payloads and related information. Options: * *
    - *
  • leveldb:path - LevelDB - *
  • mapdb:path - MapDB + *
  • leveldb:path - LevelDB
    • + *
  • mapdb:path - MapDB
    • *
  • sql:jdbcurl - Relational database
    • - *
  • memory - Contents are cleared when Orion exits + *
  • memory - Contents are cleared when Orion exits
    • *
* * Default: "leveldb" @@ -256,7 +256,7 @@ public Optional passwords() { * @return Storage string specifying a storage engine and/or storage path */ public String storage() { - return configuration.getString("storage"); + return System.getenv().getOrDefault("ORION_STORAGE", configuration.getString("storage")); } /** @@ -559,6 +559,17 @@ private static Schema configSchema() { + " - memory - Contents are cleared when Orion exits", Config::validateStorage); + schemaBuilder.addString( + "nodeStorage", + "memory", + "Storage engine used to save node information. Options:\n" + + "\n" + + " - leveldb:path - LevelDB\n" + + " - mapdb:path - MapDB\n" + + " - sql:jdbcurl - SQL database\n" + + " - memory - Contents are cleared when Orion exits", + Config::validateStorage); + schemaBuilder.addListOfString( "ipwhitelist", Collections.emptyList(), diff --git a/src/main/java/net/consensys/orion/enclave/sodium/serialization/PublicKeyURISerializer.java b/src/main/java/net/consensys/orion/enclave/sodium/serialization/PublicKeyURISerializer.java new file mode 100644 index 00000000..8071b3fc --- /dev/null +++ b/src/main/java/net/consensys/orion/enclave/sodium/serialization/PublicKeyURISerializer.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package net.consensys.orion.enclave.sodium.serialization; + +import static org.apache.tuweni.io.Base64.encodeBytes; + +import java.io.IOException; +import java.net.URI; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import org.apache.tuweni.crypto.sodium.Box; + +public class PublicKeyURISerializer extends JsonSerializer>> { + + @Override + public void serialize( + final Iterable> entries, + final JsonGenerator gen, + final SerializerProvider serializers) throws IOException { + gen.writeStartObject(); + for (Map.Entry entry : entries) { + gen.writeObjectField(encodeBytes(entry.getKey().bytesArray()), entry.getValue().toString()); + } + gen.writeEndObject(); + } +} diff --git a/src/main/java/net/consensys/orion/http/handler/knownnodes/KnownNode.java b/src/main/java/net/consensys/orion/http/handler/knownnodes/KnownNode.java index 255569b1..4edc49d3 100644 --- a/src/main/java/net/consensys/orion/http/handler/knownnodes/KnownNode.java +++ b/src/main/java/net/consensys/orion/http/handler/knownnodes/KnownNode.java @@ -12,7 +12,7 @@ */ package net.consensys.orion.http.handler.knownnodes; -import java.net.URL; +import java.net.URI; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -23,17 +23,17 @@ class KnownNode { private final String publicKey; - private final String nodeUrl; + private final String nodeURI; - KnownNode(final PublicKey publicKey, final URL nodeUrl) { + KnownNode(final PublicKey publicKey, final URI nodeURI) { this.publicKey = Base64.encodeBytes(publicKey.bytesArray()); - this.nodeUrl = nodeUrl.toString(); + this.nodeURI = nodeURI.toString(); } @JsonCreator - KnownNode(@JsonProperty("publicKey") final String publicKey, final @JsonProperty("nodeUrl") String nodeUrl) { + KnownNode(@JsonProperty("publicKey") final String publicKey, final @JsonProperty("nodeUrl") String nodeURI) { this.publicKey = publicKey; - this.nodeUrl = nodeUrl; + this.nodeURI = nodeURI; } @JsonProperty("publicKey") @@ -42,8 +42,8 @@ String getPublicKey() { } @JsonProperty("nodeUrl") - String getNodeUrl() { - return nodeUrl; + String getNodeURI() { + return nodeURI; } @Override @@ -55,11 +55,11 @@ public boolean equals(final Object o) { return false; } final KnownNode knownNode = (KnownNode) o; - return Objects.equal(publicKey, knownNode.publicKey) && Objects.equal(nodeUrl, knownNode.nodeUrl); + return Objects.equal(publicKey, knownNode.publicKey) && Objects.equal(nodeURI, knownNode.nodeURI); } @Override public int hashCode() { - return Objects.hashCode(publicKey, nodeUrl); + return Objects.hashCode(publicKey, nodeURI); } } diff --git a/src/main/java/net/consensys/orion/http/handler/knownnodes/KnownNodesHandler.java b/src/main/java/net/consensys/orion/http/handler/knownnodes/KnownNodesHandler.java index 9d832eca..adfb9024 100644 --- a/src/main/java/net/consensys/orion/http/handler/knownnodes/KnownNodesHandler.java +++ b/src/main/java/net/consensys/orion/http/handler/knownnodes/KnownNodesHandler.java @@ -13,7 +13,7 @@ package net.consensys.orion.http.handler.knownnodes; import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.network.ConcurrentNetworkNodes; +import net.consensys.orion.network.PersistentNetworkNodes; import net.consensys.orion.utils.Serializer; import java.util.ArrayList; @@ -25,9 +25,9 @@ public class KnownNodesHandler implements Handler { - private final ConcurrentNetworkNodes networkNodes; + private final PersistentNetworkNodes networkNodes; - public KnownNodesHandler(final ConcurrentNetworkNodes networkNodes) { + public KnownNodesHandler(final PersistentNetworkNodes networkNodes) { this.networkNodes = networkNodes; } @@ -35,7 +35,11 @@ public KnownNodesHandler(final ConcurrentNetworkNodes networkNodes) { public void handle(final RoutingContext routingContext) { final List knownNodes = new ArrayList<>(); - networkNodes.nodePKs().forEach((publicKey, url) -> knownNodes.add(new KnownNode(publicKey, url))); + networkNodes.nodePKs().forEach((entry) -> { + if (!entry.getValue().equals(networkNodes.uri())) { + knownNodes.add(new KnownNode(entry.getKey(), entry.getValue())); + } + }); final Buffer bufferResponse = Buffer.buffer(Serializer.serialize(HttpContentType.JSON, knownNodes)); diff --git a/src/main/java/net/consensys/orion/http/handler/partyinfo/PartyInfoHandler.java b/src/main/java/net/consensys/orion/http/handler/partyinfo/PartyInfoHandler.java index b157fd40..62381ed4 100644 --- a/src/main/java/net/consensys/orion/http/handler/partyinfo/PartyInfoHandler.java +++ b/src/main/java/net/consensys/orion/http/handler/partyinfo/PartyInfoHandler.java @@ -13,7 +13,8 @@ package net.consensys.orion.http.handler.partyinfo; import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.network.ConcurrentNetworkNodes; +import net.consensys.orion.network.PersistentNetworkNodes; +import net.consensys.orion.network.ReadOnlyNetworkNodes; import net.consensys.orion.utils.Serializer; import io.vertx.core.Handler; @@ -25,16 +26,16 @@ * knowledge of. */ public class PartyInfoHandler implements Handler { - private final ConcurrentNetworkNodes networkNodes; + private final PersistentNetworkNodes networkNodes; - public PartyInfoHandler(final ConcurrentNetworkNodes networkNodes) { + public PartyInfoHandler(final PersistentNetworkNodes networkNodes) { this.networkNodes = networkNodes; } @Override public void handle(final RoutingContext routingContext) { - final ConcurrentNetworkNodes callerPeers = - Serializer.deserialize(HttpContentType.CBOR, ConcurrentNetworkNodes.class, routingContext.getBody().getBytes()); + final ReadOnlyNetworkNodes callerPeers = + Serializer.deserialize(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, routingContext.getBody().getBytes()); final Buffer toReturn = Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, networkNodes)); routingContext.response().end(toReturn); diff --git a/src/main/java/net/consensys/orion/http/handler/privacy/CreatePrivacyGroupHandler.java b/src/main/java/net/consensys/orion/http/handler/privacy/CreatePrivacyGroupHandler.java index 5f2950c2..fa66a9e7 100644 --- a/src/main/java/net/consensys/orion/http/handler/privacy/CreatePrivacyGroupHandler.java +++ b/src/main/java/net/consensys/orion/http/handler/privacy/CreatePrivacyGroupHandler.java @@ -21,12 +21,12 @@ import net.consensys.orion.exception.OrionErrorCode; import net.consensys.orion.exception.OrionException; import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.network.ConcurrentNetworkNodes; import net.consensys.orion.network.NodeHttpClientBuilder; +import net.consensys.orion.network.PersistentNetworkNodes; import net.consensys.orion.storage.Storage; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.security.SecureRandom; import java.util.Arrays; import java.util.List; @@ -51,14 +51,14 @@ public class CreatePrivacyGroupHandler implements Handler { private final Storage privacyGroupStorage; private final Storage queryPrivacyGroupStorage; - private final ConcurrentNetworkNodes networkNodes; + private final PersistentNetworkNodes networkNodes; private final Enclave enclave; private final HttpClient httpClient; public CreatePrivacyGroupHandler( final Storage privacyGroupStorage, final Storage queryPrivacyGroupStorage, - final ConcurrentNetworkNodes networkNodes, + final PersistentNetworkNodes networkNodes, final Enclave enclave, final Vertx vertx, final Config config) { @@ -109,7 +109,7 @@ public void handle(final RoutingContext routingContext) { .map(enclave::readKey) .collect(Collectors.toList()); - if (addressListToForward.stream().anyMatch(pKey -> networkNodes.urlForRecipient(pKey) == null)) { + if (addressListToForward.stream().anyMatch(pKey -> networkNodes.uriForRecipient(pKey) == null)) { routingContext.fail(new OrionException(OrionErrorCode.NODE_MISSING_PEER_URL, "couldn't find peer URL ")); return; } @@ -118,7 +118,7 @@ public void handle(final RoutingContext routingContext) { @SuppressWarnings("rawtypes") final CompletableFuture[] cfs = addressListToForward.stream().map(pKey -> { - URL recipientURL = networkNodes.urlForRecipient(pKey); + URI recipientURL = networkNodes.uriForRecipient(pKey); log.info("Propagating create request to {} with URL {}", pKey, recipientURL.toString()); diff --git a/src/main/java/net/consensys/orion/http/handler/privacy/DeletePrivacyGroupHandler.java b/src/main/java/net/consensys/orion/http/handler/privacy/DeletePrivacyGroupHandler.java index b28e3df9..7a2efe64 100644 --- a/src/main/java/net/consensys/orion/http/handler/privacy/DeletePrivacyGroupHandler.java +++ b/src/main/java/net/consensys/orion/http/handler/privacy/DeletePrivacyGroupHandler.java @@ -21,12 +21,12 @@ import net.consensys.orion.exception.OrionErrorCode; import net.consensys.orion.exception.OrionException; import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.network.ConcurrentNetworkNodes; import net.consensys.orion.network.NodeHttpClientBuilder; +import net.consensys.orion.network.PersistentNetworkNodes; import net.consensys.orion.storage.Storage; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.util.Arrays; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -50,14 +50,14 @@ public class DeletePrivacyGroupHandler implements Handler { private final Storage privacyGroupStorage; private final Storage queryPrivacyGroupStorage; - private final ConcurrentNetworkNodes networkNodes; + private final PersistentNetworkNodes networkNodes; private final Enclave enclave; private final HttpClient httpClient; public DeletePrivacyGroupHandler( final Storage privacyGroupStorage, final Storage queryPrivacyGroupStorage, - final ConcurrentNetworkNodes networkNodes, + final PersistentNetworkNodes networkNodes, final Enclave enclave, final Vertx vertx, final Config config) { @@ -93,7 +93,7 @@ public void handle(final RoutingContext routingContext) { .map(enclave::readKey) .collect(Collectors.toList()); - if (addressListToForward.stream().anyMatch(pKey -> networkNodes.urlForRecipient(pKey) == null)) { + if (addressListToForward.stream().anyMatch(pKey -> networkNodes.uriForRecipient(pKey) == null)) { routingContext.fail(new OrionException(OrionErrorCode.NODE_MISSING_PEER_URL, "couldn't find peer URL ")); return; } @@ -102,7 +102,7 @@ public void handle(final RoutingContext routingContext) { @SuppressWarnings("rawtypes") final CompletableFuture[] cfs = addressListToForward.stream().map(pKey -> { - final URL recipientURL = networkNodes.urlForRecipient(pKey); + final URI recipientURL = networkNodes.uriForRecipient(pKey); log.info("Propagating delete request to {} with URL {}", pKey, recipientURL.toString()); diff --git a/src/main/java/net/consensys/orion/network/ConcurrentNetworkNodes.java b/src/main/java/net/consensys/orion/network/ConcurrentNetworkNodes.java deleted file mode 100644 index a4819b77..00000000 --- a/src/main/java/net/consensys/orion/network/ConcurrentNetworkNodes.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package net.consensys.orion.network; - -import net.consensys.orion.config.Config; -import net.consensys.orion.enclave.sodium.serialization.PublicKeyMapKeyDeserializer; -import net.consensys.orion.enclave.sodium.serialization.PublicKeyMapKeySerializer; - -import java.net.URL; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import org.apache.tuweni.crypto.sodium.Box; - -public class ConcurrentNetworkNodes implements NetworkNodes { - private URL url; - private final CopyOnWriteArrayList nodeURLs; - @JsonSerialize(keyUsing = PublicKeyMapKeySerializer.class) - private final ConcurrentHashMap nodePKs; - - @JsonCreator - public ConcurrentNetworkNodes( - @JsonProperty("url") final URL url, - @JsonProperty("nodeURLs") final List nodeURLs, - @JsonProperty("nodePKs") @JsonDeserialize( - keyUsing = PublicKeyMapKeyDeserializer.class) final Map nodePKs) { - this.url = url; - this.nodeURLs = new CopyOnWriteArrayList<>(nodeURLs); - this.nodePKs = new ConcurrentHashMap<>(nodePKs); - } - - public ConcurrentNetworkNodes(final Config config, final Box.PublicKey[] publicKeys) { - nodeURLs = new CopyOnWriteArrayList<>(config.otherNodes()); - nodePKs = new ConcurrentHashMap<>(); - config.nodeUrl().ifPresent(url -> setNodeUrl(url, publicKeys)); - } - - public ConcurrentNetworkNodes(final URL url) { - this(url, new CopyOnWriteArrayList<>(), new ConcurrentHashMap<>()); - } - - /** - * Set the url of the node we are running on. This is useful to do when we are running using default ports, and the - * construction of this class will be performed before we have settled on what port Orion will be running on. - * - * @param url URL of the node. - * @param publicKeys PublicKeys used by the node. - */ - public void setNodeUrl(final URL url, final Box.PublicKey[] publicKeys) { - this.url = url; - // adding my publickey(s) so /partyinfo returns my info when called. - addNodeUrl(url); - for (final Box.PublicKey publicKey : publicKeys) { - nodePKs.put(publicKey, url); - } - } - - /** - * Add a node's URL and public keys to the nodeURLs and nodePKs lists - * - * @param nodesPks List of PublicKeys of new node - * @param node URL of new node - */ - public void addNode(final List nodesPks, final URL node) { - this.nodeURLs.addIfAbsent(node); - nodesPks.forEach(nodePk -> this.nodePKs.putIfAbsent(nodePk, node)); - } - - /** - * Add a url of a node to perform discovery using that node. - * - * @param node URL of new node - */ - public void addNodeUrl(final URL node) { - this.nodeURLs.addIfAbsent(node); - } - - @Override - @JsonProperty("url") - public URL url() { - return url; - } - - @Override - @JsonProperty("nodeURLs") - public Collection nodeURLs() { - return nodeURLs; - } - - @Override - public URL urlForRecipient(final Box.PublicKey recipient) { - return nodePKs.get(recipient); - } - - @Override - @JsonProperty("nodePKs") - public Map nodePKs() { - return nodePKs; - } - - @Override - public boolean merge(final ConcurrentNetworkNodes other) { - // note; not using map.putAll() as we don't want a malicious peer to overwrite ours nodes. - boolean thisChanged = false; - - for (final Map.Entry entry : other.nodePKs().entrySet()) { - if (nodePKs.putIfAbsent(entry.getKey(), entry.getValue()) == null) { - // putIfAbsent returns null if there was no mapping associated with the provided key - thisChanged = true; - nodeURLs.addIfAbsent(entry.getValue()); - } - } - return thisChanged; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - final ConcurrentNetworkNodes that = (ConcurrentNetworkNodes) o; - - return Objects.equals(url, that.url) - && Objects.equals(nodeURLs, that.nodeURLs) - && Objects.equals(nodePKs, that.nodePKs); - } - - @Override - public int hashCode() { - return Objects.hash(url, nodeURLs, nodePKs); - } -} diff --git a/src/main/java/net/consensys/orion/network/NetworkDiscovery.java b/src/main/java/net/consensys/orion/network/NetworkDiscovery.java index 67b5f0ac..c30b31a1 100644 --- a/src/main/java/net/consensys/orion/network/NetworkDiscovery.java +++ b/src/main/java/net/consensys/orion/network/NetworkDiscovery.java @@ -17,7 +17,7 @@ import net.consensys.orion.config.Config; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.time.Instant; import java.util.HashMap; import java.util.Map; @@ -39,14 +39,14 @@ public class NetworkDiscovery extends AbstractVerticle { private Optional httpClient = Optional.empty(); - private final ConcurrentNetworkNodes nodes; - private final Map discoverers; + private final PersistentNetworkNodes nodes; + private final Map discoverers; private final Config config; private final long refreshDelayMs; private final int clientTimeoutMs; public NetworkDiscovery( - final ConcurrentNetworkNodes nodes, + final PersistentNetworkNodes nodes, final Config config, final long refreshDelayMs, final int clientTimeoutMs) { @@ -57,7 +57,7 @@ public NetworkDiscovery( this.clientTimeoutMs = clientTimeoutMs; } - public NetworkDiscovery(final ConcurrentNetworkNodes nodes, final Config config) { + public NetworkDiscovery(final PersistentNetworkNodes nodes, final Config config) { this(nodes, config, REFRESH_DELAY_MS, HTTP_CLIENT_TIMEOUT_MS); } @@ -84,25 +84,19 @@ public void stop() { */ private void updateDiscoverers() { // for each peer that we know, we start a Discoverer (on timer) - for (final URL nodeUrl : nodes.nodeURLs()) { - final String urlString = nodeUrl.toString(); - if (!discoverers.containsKey(urlString)) { - final Discoverer d = new Discoverer(nodeUrl, refreshDelayMs, nodes.url().equals(nodeUrl)); - discoverers.put(urlString, d); + for (final URI nodeUri : nodes.nodeURIs()) { + if (!(nodeUri.equals(nodes.uri()) || discoverers.containsKey(nodeUri))) { + final Discoverer d = new Discoverer(nodeUri, refreshDelayMs, nodes.uri().equals(nodeUri)); + discoverers.put(nodeUri, d); d.engageNextTimerTick(); } } } - public Map discoverers() { + public Map discoverers() { return new HashMap<>(discoverers); } - public void addPeer(final URL url) { - nodes.addNodeUrl(url); - updateDiscoverers(); - } - /** * Discoverer handle() is fired by a timer * @@ -110,14 +104,14 @@ public void addPeer(final URL url) { * Its job is to call /partyInfo periodically on a specified URL and merge results if needed in NetworkDiscovery state */ class Discoverer implements Handler { - private final URL nodeUrl; + private final URI nodeUrl; long currentRefreshDelay; Instant lastUpdate = Instant.MIN; long attempts = 0; private long timerId; private final boolean self; - Discoverer(final URL nodeUrl, final long refreshDelayMs, final boolean self) { + Discoverer(final URI nodeUrl, final long refreshDelayMs, final boolean self) { this.nodeUrl = nodeUrl; this.currentRefreshDelay = refreshDelayMs; this.self = self; @@ -141,11 +135,10 @@ public void handle(final Long timerId) { .post(nodeUrl.getPort(), nodeUrl.getHost(), "/partyinfo", resp -> { if (resp.statusCode() == 200) { lastUpdate = Instant.now(); - resp.bodyHandler(respBody -> { // deserialize response - ConcurrentNetworkNodes partyInfoResponse = - Serializer.deserialize(CBOR, ConcurrentNetworkNodes.class, respBody.getBytes()); + ReadOnlyNetworkNodes partyInfoResponse = + Serializer.deserialize(CBOR, ReadOnlyNetworkNodes.class, respBody.getBytes()); if (nodes.merge(partyInfoResponse)) { log.info("merged new nodes from {} discoverer", nodeUrl); } @@ -158,6 +151,7 @@ public void handle(final Long timerId) { }) .exceptionHandler(ex -> { log.error("calling partyInfo on {} failed {}", nodeUrl, ex.getMessage()); + log.error(ex.getMessage(), ex); engageNextTimerTick(); }) .putHeader("Content-Type", "application/cbor") diff --git a/src/main/java/net/consensys/orion/network/NetworkNodes.java b/src/main/java/net/consensys/orion/network/NetworkNodes.java index 7f1bf42b..839d1189 100644 --- a/src/main/java/net/consensys/orion/network/NetworkNodes.java +++ b/src/main/java/net/consensys/orion/network/NetworkNodes.java @@ -12,10 +12,14 @@ */ package net.consensys.orion.network; -import java.net.URL; +import net.consensys.orion.enclave.sodium.serialization.PublicKeyURISerializer; + +import java.net.URI; import java.util.Collection; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import org.apache.tuweni.crypto.sodium.Box; /** Details of other nodes on the network */ @@ -24,19 +28,27 @@ public interface NetworkNodes { /** * @return URL of node */ - URL url(); + @JsonProperty("uri") + URI uri(); /** - * @return List of URLs of other nodes on the network + * @return List of URIs of other nodes on the network */ - Collection nodeURLs(); + @JsonProperty("nodeURIs") + Collection nodeURIs(); - URL urlForRecipient(Box.PublicKey recipient); + /** + * Provide the URI associated with a public key. + * + * @param recipient the public key of the recipient of a message + * @return the URI, or null if no recipient exists for the given URI. + */ + URI uriForRecipient(Box.PublicKey recipient); /** * @return Map from public key to node for all discovered nodes. */ - Map nodePKs(); - - boolean merge(ConcurrentNetworkNodes other); + @JsonProperty("nodePKs") + @JsonSerialize(using = PublicKeyURISerializer.class) + Iterable> nodePKs(); } diff --git a/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java b/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java new file mode 100644 index 00000000..0040d94f --- /dev/null +++ b/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java @@ -0,0 +1,191 @@ +/* + * Copyright 2018 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package net.consensys.orion.network; + +import net.consensys.orion.config.Config; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.tuweni.concurrent.AsyncCompletion; +import org.apache.tuweni.crypto.sodium.Box; +import org.apache.tuweni.kv.KeyValueStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PersistentNetworkNodes implements NetworkNodes { + private final static Logger logger = LoggerFactory.getLogger(PersistentNetworkNodes.class); + private URI uri; + private final KeyValueStore nodePKs; + + public PersistentNetworkNodes( + final Config config, + final Box.PublicKey[] publicKeys, + final KeyValueStore store) { + nodePKs = store; + config.nodeUrl().ifPresent(nodeURL -> { + try { + setNodeUrl(nodeURL.toURI(), publicKeys); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + }); + } + + /** + * Set the url of the node we are running on. This is useful to do when we are running using default ports, and the + * construction of this class will be performed before we have settled on what port Orion will be running on. + * + * @param uri URI of the node. + * @param publicKeys PublicKeys used by the node. + */ + public void setNodeUrl(final URI uri, final Box.PublicKey[] publicKeys) { + this.uri = uri; + try { + AsyncCompletion.allOf(Arrays.stream(publicKeys).map(pk -> nodePKs.putAsync(pk, uri))).join(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + /** + * Add a node's URL and public keys to the nodeURLs and nodePKs lists + * + * @param nodesPks List of PublicKeys of new node + */ + public boolean addNode(final Iterable> nodesPks) { + AtomicBoolean changed = new AtomicBoolean(false); + List completions = new ArrayList<>(); + nodesPks.forEach(entry -> { + Box.PublicKey nodePk = entry.getKey(); + URI nodeURI = entry.getValue(); + completions.add(nodePKs.getAsync(nodePk).thenAccept(oldNodeURL -> { + if (oldNodeURL == null) { + nodePKs.putAsync(nodePk, nodeURI); + changed.set(true); + } + })); + }); + try { + AsyncCompletion.allOf(completions).join(); + } catch (InterruptedException e) { + logger.warn("Timeout waiting to store node URL", e); + } + return changed.get(); + } + + @Override + public URI uri() { + return uri; + } + + @Override + public Collection nodeURIs() { + Set allURIs = new HashSet<>(); + List completions = new ArrayList<>(); + nodePKs.keysAsync().thenAccept(keys -> { + for (Box.PublicKey key : keys) { + completions.add(nodePKs.getAsync(key).thenAccept(allURIs::add)); + } + }); + try { + AsyncCompletion.allOf(completions).join(); + } catch (InterruptedException e) { + logger.warn("Timeout waiting to collect all URLs", e); + } + return allURIs; + } + + @Override + public URI uriForRecipient(final Box.PublicKey recipient) { + try { + return nodePKs.getAsync(recipient).get(); + } catch (InterruptedException e) { + logger.warn("Timeout waiting to retrieve URL information", e); + return null; + } + } + + @Override + public Iterable> nodePKs() { + try { + Iterator iter = nodePKs.keysAsync().get().iterator(); + return () -> new Iterator<>() { + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public Map.Entry next() { + Box.PublicKey key = iter.next(); + return new Map.Entry<>() { + + @Override + public Box.PublicKey getKey() { + return key; + } + + @Override + public URI getValue() { + return uriForRecipient(key); + } + + @Override + public URI setValue(URI value) { + throw new UnsupportedOperationException(); + } + }; + } + }; + } catch (InterruptedException e) { + logger.warn("Timeout waiting to retrieve keys", e); + return null; + } + } + + /** + * Merge with another set of node information. + * + * @param other the other set of nodes + * @return true if we updated our records. + */ + public boolean merge(final NetworkNodes other) { + return addNode(other.nodePKs()); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof PersistentNetworkNodes)) + return false; + PersistentNetworkNodes that = (PersistentNetworkNodes) o; + return Objects.equals(uri, that.uri) && Objects.equals(nodePKs, that.nodePKs); + } + + @Override + public int hashCode() { + return Objects.hash(uri, nodePKs); + } +} diff --git a/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java b/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java new file mode 100644 index 00000000..99cc296f --- /dev/null +++ b/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java @@ -0,0 +1,81 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package net.consensys.orion.network; + +import net.consensys.orion.enclave.sodium.serialization.PublicKeyMapKeyDeserializer; + +import java.net.URI; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.apache.tuweni.crypto.sodium.Box; + +public class ReadOnlyNetworkNodes implements NetworkNodes { + + private final URI uri; + private final CopyOnWriteArrayList nodeURIs; + private final Map nodePKs; + + @JsonCreator + public ReadOnlyNetworkNodes( + @JsonProperty("uri") final URI uri, + @JsonProperty("nodeURIs") final List nodeURIs, + @JsonProperty("nodePKs") @JsonDeserialize( + keyUsing = PublicKeyMapKeyDeserializer.class) final Map nodePKs) { + this.uri = uri; + this.nodeURIs = new CopyOnWriteArrayList<>(nodeURIs); + this.nodePKs = new HashMap<>(nodePKs); + } + + @Override + public URI uri() { + return uri; + } + + @Override + public Collection nodeURIs() { + return nodeURIs; + } + + @Override + public Iterable> nodePKs() { + return nodePKs.entrySet(); + } + + @Override + public URI uriForRecipient(final Box.PublicKey recipient) { + return nodePKs.get(recipient); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ReadOnlyNetworkNodes that = (ReadOnlyNetworkNodes) o; + return uri.equals(that.uri) && nodeURIs.equals(that.nodeURIs) && nodePKs.equals(that.nodePKs); + } + + @Override + public int hashCode() { + return Objects.hash(uri, nodeURIs, nodePKs); + } +} diff --git a/src/main/java/net/consensys/orion/payload/DistributePayloadManager.java b/src/main/java/net/consensys/orion/payload/DistributePayloadManager.java index 6d0a9902..e1379eda 100644 --- a/src/main/java/net/consensys/orion/payload/DistributePayloadManager.java +++ b/src/main/java/net/consensys/orion/payload/DistributePayloadManager.java @@ -22,12 +22,12 @@ import net.consensys.orion.http.handler.send.SendRequest; import net.consensys.orion.http.handler.send.SendResponse; import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.network.ConcurrentNetworkNodes; import net.consensys.orion.network.NodeHttpClientBuilder; +import net.consensys.orion.network.PersistentNetworkNodes; import net.consensys.orion.storage.Storage; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -58,7 +58,7 @@ public class DistributePayloadManager { private final Storage storage; private final Storage privacyGroupStorage; private final Storage queryPrivacyGroupStorage; - private final ConcurrentNetworkNodes networkNodes; + private final PersistentNetworkNodes networkNodes; private final List nodeKeys; private final HttpClient httpClient; @@ -69,7 +69,7 @@ public DistributePayloadManager( final Storage storage, final Storage privacyGroupStorage, final Storage queryPrivacyGroupStorage, - final ConcurrentNetworkNodes networkNodes) { + final PersistentNetworkNodes networkNodes) { this( enclave, storage, @@ -85,7 +85,7 @@ public DistributePayloadManager( final Storage storage, final Storage privacyGroupStorage, final Storage queryPrivacyGroupStorage, - final ConcurrentNetworkNodes networkNodes, + final PersistentNetworkNodes networkNodes, final HttpClient httpClient) { this.enclave = enclave; this.storage = storage; @@ -223,25 +223,24 @@ private Future sendPayloadToParticipants( final List keys = toKeys.stream().filter(pKey -> !nodeKeys.contains(pKey)).collect(Collectors.toList()); - if (keys.stream().anyMatch(pKey -> networkNodes.urlForRecipient(pKey) == null)) { + if (keys.stream().anyMatch(pKey -> networkNodes.uriForRecipient(pKey) == null)) { throw new OrionException(OrionErrorCode.NODE_MISSING_PEER_URL, "couldn't find peer URL"); } - @SuppressWarnings("URLEqualsHashCode") - final Map> urlToKeysMap = getUrlToKeyListMap(keys); + final Map> uriToKeysMap = getUriToKeyListMap(keys); log.debug("Generate payload digest"); final String digest = storage.generateDigest(encryptedPayload); log.debug("propagating payload"); @SuppressWarnings("rawtypes") - final CompletableFuture[] cfs = urlToKeysMap.keySet().stream().map(url -> { + final CompletableFuture[] cfs = uriToKeysMap.keySet().stream().map(url -> { CompletableFuture responseFuture = new CompletableFuture<>(); // serialize payload, stripping non-relevant encryptedKeys, and configureRoutes payload final byte[] payload = - Serializer.serialize(HttpContentType.CBOR, encryptedPayload.stripFor(urlToKeysMap.get(url))); + Serializer.serialize(HttpContentType.CBOR, encryptedPayload.stripFor(uriToKeysMap.get(url))); // execute request httpClient @@ -278,11 +277,11 @@ private Future sendPayloadToParticipants( } @NotNull - private Map> getUrlToKeyListMap(final List keys) { + private Map> getUriToKeyListMap(final List keys) { @SuppressWarnings("URLEqualsHashCode") - final Map> urlToKeysMap = new HashMap<>(); + final Map> urlToKeysMap = new HashMap<>(); for (final PublicKey key : keys) { - final URL url = networkNodes.urlForRecipient(key); + final URI url = networkNodes.uriForRecipient(key); if (!urlToKeysMap.containsKey(url)) { urlToKeysMap.put(url, new ArrayList<>()); } diff --git a/src/main/java/net/consensys/orion/storage/EncryptedPayloadStorage.java b/src/main/java/net/consensys/orion/storage/EncryptedPayloadStorage.java index 6ae24e49..9fb03a63 100644 --- a/src/main/java/net/consensys/orion/storage/EncryptedPayloadStorage.java +++ b/src/main/java/net/consensys/orion/storage/EncryptedPayloadStorage.java @@ -29,10 +29,10 @@ public class EncryptedPayloadStorage implements Storage { - private final KeyValueStore store; + private final KeyValueStore store; private final StorageKeyBuilder keyBuilder; - public EncryptedPayloadStorage(final KeyValueStore store, final StorageKeyBuilder keyBuilder) { + public EncryptedPayloadStorage(final KeyValueStore store, final StorageKeyBuilder keyBuilder) { this.store = store; this.keyBuilder = keyBuilder; } diff --git a/src/main/java/net/consensys/orion/storage/OrionSQLKeyValueStore.java b/src/main/java/net/consensys/orion/storage/OrionSQLKeyValueStore.java deleted file mode 100644 index f989d292..00000000 --- a/src/main/java/net/consensys/orion/storage/OrionSQLKeyValueStore.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package net.consensys.orion.storage; - -import java.util.Base64; -import java.util.function.Function; -import javax.persistence.EntityManager; - -import kotlin.Unit; -import kotlin.coroutines.Continuation; -import kotlinx.coroutines.CoroutineDispatcher; -import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.concurrent.AsyncCompletion; -import org.apache.tuweni.concurrent.AsyncResult; -import org.apache.tuweni.kv.KeyValueStore; - -public class OrionSQLKeyValueStore implements KeyValueStore { - private final JpaEntityManagerProvider jpaEntityManagerProvider; - - public OrionSQLKeyValueStore(final JpaEntityManagerProvider jpaEntityManagerProvider) { - this.jpaEntityManagerProvider = jpaEntityManagerProvider; - } - - private T withEntityManager(final Function entityManagerConsumer) { - final EntityManager entityManager = jpaEntityManagerProvider.createEntityManager(); - try { - entityManager.getTransaction().begin(); - return entityManagerConsumer.apply(entityManager); - } finally { - entityManager.getTransaction().commit(); - entityManager.close(); - } - } - - @Override - public Bytes get(final Bytes key, final Continuation ignore) { - return withEntityManager(entityManager -> { - final String b64String = Base64.getEncoder().encodeToString(key.toArrayUnsafe()); - final Store store = entityManager.find(Store.class, b64String); - return store != null ? Bytes.wrap(store.getValue()) : null; - }); - } - - @Override - public AsyncResult getAsync(final CoroutineDispatcher ignore, final Bytes key) { - return AsyncResult.executeBlocking(() -> get(key, null)); - } - - @Override - public AsyncResult getAsync(final Bytes key) { - return AsyncResult.executeBlocking(() -> get(key, null)); - } - - @Override - public Unit put(final Bytes key, final Bytes value, final Continuation ignore) { - final Store store = new Store(); - final String b64String = Base64.getEncoder().encodeToString(key.toArrayUnsafe()); - store.setKey(b64String); - store.setValue(value.toArrayUnsafe()); - withEntityManager(entityManager -> entityManager.merge(store)); - return Unit.INSTANCE; - } - - @Override - public AsyncCompletion putAsync(final CoroutineDispatcher ignore, final Bytes key, final Bytes value) { - return AsyncCompletion.executeBlocking(() -> put(key, value, null)); - } - - @Override - public AsyncCompletion putAsync(final Bytes key, final Bytes value) { - return AsyncCompletion.executeBlocking(() -> put(key, value, null)); - } - - @Override - public void close() { - jpaEntityManagerProvider.close(); - } -} diff --git a/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java b/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java index be1eb2c7..3c40c849 100644 --- a/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java +++ b/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java @@ -33,10 +33,10 @@ public class PrivacyGroupStorage implements Storage { - private final KeyValueStore store; + private final KeyValueStore store; private final Enclave enclave; - public PrivacyGroupStorage(final KeyValueStore store, final Enclave enclave) { + public PrivacyGroupStorage(final KeyValueStore store, final Enclave enclave) { this.store = store; this.enclave = enclave; } diff --git a/src/main/java/net/consensys/orion/storage/QueryPrivacyGroupStorage.java b/src/main/java/net/consensys/orion/storage/QueryPrivacyGroupStorage.java index 22e79d92..c5463588 100644 --- a/src/main/java/net/consensys/orion/storage/QueryPrivacyGroupStorage.java +++ b/src/main/java/net/consensys/orion/storage/QueryPrivacyGroupStorage.java @@ -34,10 +34,10 @@ public class QueryPrivacyGroupStorage implements Storage { private static final byte[] BYTES = Bytes.fromHexString("5375ba871e5c3d0f1d055b5da0ac02ea035bed38").toArrayUnsafe(); - private final KeyValueStore store; + private final KeyValueStore store; private final Enclave enclave; - public QueryPrivacyGroupStorage(final KeyValueStore store, final Enclave enclave) { + public QueryPrivacyGroupStorage(final KeyValueStore store, final Enclave enclave) { this.store = store; this.enclave = enclave; } diff --git a/src/test/java/net/consensys/orion/helpers/FakePeer.java b/src/test/java/net/consensys/orion/helpers/FakePeer.java index a44d4faa..aa08aafc 100644 --- a/src/test/java/net/consensys/orion/helpers/FakePeer.java +++ b/src/test/java/net/consensys/orion/helpers/FakePeer.java @@ -15,7 +15,7 @@ import net.consensys.orion.enclave.sodium.MemoryKeyStore; import java.io.IOException; -import java.net.URL; +import java.net.URI; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -50,7 +50,7 @@ public void addResponse(final MockResponse response) { server.enqueue(response); } - public URL getURL() { - return server.url("").url(); + public URI getURI() { + return server.url("").uri(); } } diff --git a/src/test/java/net/consensys/orion/http/handler/CreatePrivacyGroupHandlerTest.java b/src/test/java/net/consensys/orion/http/handler/CreatePrivacyGroupHandlerTest.java index a9b6ed1a..d5ae67e5 100644 --- a/src/test/java/net/consensys/orion/http/handler/CreatePrivacyGroupHandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/CreatePrivacyGroupHandlerTest.java @@ -25,7 +25,7 @@ import net.consensys.orion.utils.Serializer; import java.io.IOException; -import java.net.URL; +import java.net.URI; import java.nio.file.Path; import java.security.SecureRandom; import java.util.Arrays; @@ -69,7 +69,7 @@ void expectedPrivacyGroupId() throws Exception { // create fake peer final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(encodeBytes(privacyGroupPayload)), recipientKey); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); // execute request final Response resp = httpClient.newCall(request).execute(); @@ -108,15 +108,15 @@ void oddNumberOfRecipientsPrivacyGroupId() throws IOException { // create fake peers final FakePeer fakePeer1 = new FakePeer(new MockResponse().setBody(encodeBytes(privacyGroupPayload)), recipientKey1); - networkNodes.addNode(Collections.singletonList(fakePeer1.publicKey), fakePeer1.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer1.publicKey, fakePeer1.getURI()).entrySet()); final FakePeer fakePeer2 = new FakePeer(new MockResponse().setBody(encodeBytes(privacyGroupPayload)), recipientKey2); - networkNodes.addNode(Collections.singletonList(fakePeer2.publicKey), fakePeer2.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer2.publicKey, fakePeer2.getURI()).entrySet()); final FakePeer fakePeer3 = new FakePeer(new MockResponse().setBody(encodeBytes(privacyGroupPayload)), recipientKey3); - networkNodes.addNode(Collections.singletonList(fakePeer3.publicKey), fakePeer3.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer3.publicKey, fakePeer3.getURI()).entrySet()); // execute request final Response resp = httpClient.newCall(request).execute(); @@ -152,7 +152,7 @@ void RepeatedRecipientsPrivacyGroupId() throws IOException { // create fake peer final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(encodeBytes(privacyGroupPayload)), recipientKey); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); // execute request final Response resp = httpClient.newCall(request).execute(); @@ -216,7 +216,7 @@ void expectedPrivacyGroupIdWithoutOptionalParameters() throws Exception { // create fake peer final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(encodeBytes(privacyGroupPayload)), recipientKey); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); // execute request final Response resp = httpClient.newCall(request).execute(); @@ -246,8 +246,8 @@ class FakePeer { server.start(); } - URL getURL() { - return server.url("").url(); + URI getURI() { + return server.url("").uri(); } } } diff --git a/src/test/java/net/consensys/orion/http/handler/DeletePrivacyGroupHandlerTest.java b/src/test/java/net/consensys/orion/http/handler/DeletePrivacyGroupHandlerTest.java index 2ccbbda8..910b43c6 100644 --- a/src/test/java/net/consensys/orion/http/handler/DeletePrivacyGroupHandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/DeletePrivacyGroupHandlerTest.java @@ -71,7 +71,7 @@ void setup() throws IOException, InterruptedException { // create fake peer fakePeer = new FakePeer(new MockResponse().setBody(encodeBytes(privacyGroupPayload)), recipientKey); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); // execute request final Response resp = httpClient.newCall(request).execute(); diff --git a/src/test/java/net/consensys/orion/http/handler/FindPrivacyGroupHandlerTest.java b/src/test/java/net/consensys/orion/http/handler/FindPrivacyGroupHandlerTest.java index 3afbd1b6..b78c2405 100644 --- a/src/test/java/net/consensys/orion/http/handler/FindPrivacyGroupHandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/FindPrivacyGroupHandlerTest.java @@ -76,7 +76,7 @@ void setup() throws IOException, InterruptedException { // create fake peer fakePeer = new FakePeer(new MockResponse().setBody(encodeBytes(privacyGroupPayload)), recipientKey); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); // execute request final Response resp = httpClient.newCall(request).execute(); diff --git a/src/test/java/net/consensys/orion/http/handler/HandlerTest.java b/src/test/java/net/consensys/orion/http/handler/HandlerTest.java index c6a5be4c..73b56c99 100644 --- a/src/test/java/net/consensys/orion/http/handler/HandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/HandlerTest.java @@ -23,7 +23,7 @@ import net.consensys.orion.exception.OrionErrorCode; import net.consensys.orion.helpers.StubEnclave; import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.network.ConcurrentNetworkNodes; +import net.consensys.orion.network.PersistentNetworkNodes; import net.consensys.orion.payload.DistributePayloadManager; import net.consensys.orion.storage.EncryptedPayloadStorage; import net.consensys.orion.storage.PrivacyGroupStorage; @@ -47,12 +47,14 @@ import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.concurrent.AsyncCompletion; import org.apache.tuweni.concurrent.CompletableAsyncCompletion; +import org.apache.tuweni.crypto.sodium.Box; import org.apache.tuweni.junit.TempDirectory; import org.apache.tuweni.junit.TempDirectoryExtension; import org.apache.tuweni.kv.KeyValueStore; -import org.apache.tuweni.kv.MapDBKeyValueStore; +import org.apache.tuweni.kv.MapKeyValueStore; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.ExtendWith; @@ -71,7 +73,7 @@ public abstract class HandlerTest { protected String clientBaseUrl; // these are re-built between tests - protected ConcurrentNetworkNodes networkNodes; + protected PersistentNetworkNodes networkNodes; protected Config config; protected Enclave enclave; @@ -81,7 +83,7 @@ public abstract class HandlerTest { private Integer clientHTTPServerPort; private HttpServer clientHttpServer; - private KeyValueStore storage; + private KeyValueStore storage; protected Storage payloadStorage; protected Storage queryPrivacyGroupStorage; protected Storage privacyGroupStorage; @@ -99,11 +101,11 @@ void setUp(@TempDirectory final Path tempDir) throws Exception { // orion dependencies, reset them all between tests config = Config.load("tls='off'\nworkdir=\"" + tempDir + "\""); - networkNodes = new ConcurrentNetworkNodes(nodeHTTP.url()); + storage = MapKeyValueStore.open(); + networkNodes = + new PersistentNetworkNodes(config, new Box.PublicKey[] {Box.KeyPair.random().publicKey()}, Orion.wrap(storage)); enclave = buildEnclave(tempDir); - final Path path = tempDir.resolve("routerdb"); - storage = MapDBKeyValueStore.open(path); // create our vertx object vertx = Vertx.vertx(); final StorageKeyBuilder keyBuilder = new Sha512_256StorageKeyBuilder(); diff --git a/src/test/java/net/consensys/orion/http/handler/PartyInfoHandlerTest.java b/src/test/java/net/consensys/orion/http/handler/PartyInfoHandlerTest.java index 90416ef2..c81539aa 100644 --- a/src/test/java/net/consensys/orion/http/handler/PartyInfoHandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/PartyInfoHandlerTest.java @@ -15,13 +15,14 @@ import static net.consensys.orion.http.server.HttpContentType.CBOR; import static net.consensys.orion.http.server.HttpContentType.JSON; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import net.consensys.orion.exception.OrionErrorCode; import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.network.ConcurrentNetworkNodes; +import net.consensys.orion.network.ReadOnlyNetworkNodes; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.util.Collections; import okhttp3.MediaType; @@ -35,10 +36,10 @@ class PartyInfoHandlerTest extends HandlerTest { @Test void successfulProcessingOfRequest() throws Exception { - networkNodes - .addNode(Collections.singletonList(Box.KeyPair.random().publicKey()), new URL("http://127.0.0.1:9001/")); - networkNodes - .addNode(Collections.singletonList(Box.KeyPair.random().publicKey()), new URL("http://127.0.0.1:9002/")); + networkNodes.addNode( + Collections.singletonMap(Box.KeyPair.random().publicKey(), URI.create("http://127.0.0.1:9001/")).entrySet()); + networkNodes.addNode( + Collections.singletonMap(Box.KeyPair.random().publicKey(), URI.create("http://127.0.0.1:9002/")).entrySet()); // prepare /partyinfo payload (our known peers) final RequestBody partyInfoBody = @@ -50,26 +51,29 @@ void successfulProcessingOfRequest() throws Exception { final Response resp = httpClient.newCall(request).execute(); assertEquals(200, resp.code()); - final ConcurrentNetworkNodes partyInfoResponse = - Serializer.deserialize(HttpContentType.CBOR, ConcurrentNetworkNodes.class, resp.body().bytes()); + final ReadOnlyNetworkNodes partyInfoResponse = + Serializer.deserialize(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, resp.body().bytes()); - assertEquals(networkNodes, partyInfoResponse); + assertEquals(networkNodes.uri(), partyInfoResponse.uri()); + assertFalse(networkNodes.merge(partyInfoResponse)); } @Test void roundTripSerialization() throws Exception { - final ConcurrentNetworkNodes networkNodes = new ConcurrentNetworkNodes(new URL("http://localhost:1234/")); - networkNodes.addNode(Collections.singletonList(Box.KeyPair.random().publicKey()), new URL("http://localhost/")); - assertEquals(networkNodes, Serializer.roundTrip(HttpContentType.CBOR, ConcurrentNetworkNodes.class, networkNodes)); - assertEquals(networkNodes, Serializer.roundTrip(HttpContentType.JSON, ConcurrentNetworkNodes.class, networkNodes)); + final ReadOnlyNetworkNodes networkNodes = new ReadOnlyNetworkNodes( + URI.create("http://localhost:1234/"), + Collections.singletonList(URI.create("http://localhost/")), + Collections.singletonMap(Box.KeyPair.random().publicKey(), URI.create("http://localhost/"))); + assertEquals(networkNodes, Serializer.roundTrip(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, networkNodes)); + assertEquals(networkNodes, Serializer.roundTrip(HttpContentType.JSON, ReadOnlyNetworkNodes.class, networkNodes)); } @Test void partyInfoWithInvalidContentType() throws Exception { - networkNodes - .addNode(Collections.singletonList(Box.KeyPair.random().publicKey()), new URL("http://127.0.0.1:9001/")); - networkNodes - .addNode(Collections.singletonList(Box.KeyPair.random().publicKey()), new URL("http://127.0.0.1:9002/")); + networkNodes.addNode( + Collections.singletonMap(Box.KeyPair.random().publicKey(), URI.create("http://127.0.0.1:9001/")).entrySet()); + networkNodes.addNode( + Collections.singletonMap(Box.KeyPair.random().publicKey(), URI.create("http://127.0.0.1:9002/")).entrySet()); // prepare /partyinfo payload (our known peers) with invalid content type (json) final RequestBody partyInfoBody = diff --git a/src/test/java/net/consensys/orion/http/handler/RetrievePrivacyGroupHandlerTest.java b/src/test/java/net/consensys/orion/http/handler/RetrievePrivacyGroupHandlerTest.java index cfcec0b7..d3b4597f 100644 --- a/src/test/java/net/consensys/orion/http/handler/RetrievePrivacyGroupHandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/RetrievePrivacyGroupHandlerTest.java @@ -82,7 +82,7 @@ private String createPrivacyGroupId(final PublicKey recipientKey) throws IOExcep PrivacyGroupPayload.Type.PANTHEON); peer.addResponse(new MockResponse().setBody(encodeBytes(privacyGroupId))); - networkNodes.addNode(Collections.singletonList(peer.publicKey), peer.getURL()); + networkNodes.addNode(Collections.singletonMap(peer.publicKey, peer.getURI()).entrySet()); final Response resp = httpClient.newCall(request).execute(); diff --git a/src/test/java/net/consensys/orion/http/handler/SendHandlerTest.java b/src/test/java/net/consensys/orion/http/handler/SendHandlerTest.java index 8925ebb5..e334c10e 100644 --- a/src/test/java/net/consensys/orion/http/handler/SendHandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/SendHandlerTest.java @@ -93,7 +93,7 @@ void sendApiOnlyWorksOnPrivatePort() throws Exception { // create fake peer final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(digest), memoryKeyStore); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); // configureRoutes our sendRequest final Map sendRequest = buildRequest(Collections.singletonList(fakePeer), toEncrypt); diff --git a/src/test/java/net/consensys/orion/http/handler/SendRawHandlerTest.java b/src/test/java/net/consensys/orion/http/handler/SendRawHandlerTest.java index 219d8ebd..33bad384 100644 --- a/src/test/java/net/consensys/orion/http/handler/SendRawHandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/SendRawHandlerTest.java @@ -66,7 +66,7 @@ void sendingWithARawBody() throws Exception { for (int i = 0; i < 5; i++) { final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(digest), memoryKeyStore); // add peer push URL to networkNodes - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); fakePeers.add(fakePeer); } @@ -107,7 +107,7 @@ void sendRawApiOnlyWorksOnPrivatePort() throws Exception { // create fake peers final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(digest), memoryKeyStore); // add peer push URL to networkNodes - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); // configureRoutes the binary sendRequest final RequestBody body = diff --git a/src/test/java/net/consensys/orion/http/handler/SendRawHandlerWithNodeKeys.java b/src/test/java/net/consensys/orion/http/handler/SendRawHandlerWithNodeKeys.java index 46247bac..778612a8 100644 --- a/src/test/java/net/consensys/orion/http/handler/SendRawHandlerWithNodeKeys.java +++ b/src/test/java/net/consensys/orion/http/handler/SendRawHandlerWithNodeKeys.java @@ -64,7 +64,7 @@ public void sendingWithoutFromHeaderSucceeds() throws Exception { final String digest = encodeBytes(sha2_512_256(encryptedPayload.cipherText())); final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(digest), memoryKeyStore); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); final String[] to = new String[] {encodeBytes(fakePeer.publicKey.bytesArray())}; final Request request = new Request.Builder() diff --git a/src/test/java/net/consensys/orion/http/handler/knownnodes/KnownNodesHandlerTest.java b/src/test/java/net/consensys/orion/http/handler/knownnodes/KnownNodesHandlerTest.java index c86131ad..979329b2 100644 --- a/src/test/java/net/consensys/orion/http/handler/knownnodes/KnownNodesHandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/knownnodes/KnownNodesHandlerTest.java @@ -18,8 +18,7 @@ import net.consensys.orion.http.handler.HandlerTest; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.URI; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -55,7 +54,7 @@ void shouldReturnEmptyListWhenOrionHasNoNodes() throws Exception { final Response response = httpClient.newCall(request).execute(); final List knownNodes = readList(response); - assertTrue(knownNodes.isEmpty()); + assertTrue(knownNodes.isEmpty(), () -> knownNodes.get(0).getNodeURI()); } @Test @@ -81,8 +80,8 @@ private List readList(final Response response) throws java.io.IOExcep private List createNodes() { try { final List knownNodes = new ArrayList<>(); - knownNodes.add(new KnownNode(KeyPair.random().publicKey(), new URL("http://127.0.0.1:9001/"))); - knownNodes.add(new KnownNode(KeyPair.random().publicKey(), new URL("http://127.0.0.1:9002/"))); + knownNodes.add(new KnownNode(KeyPair.random().publicKey(), URI.create("http://127.0.0.1:9001/"))); + knownNodes.add(new KnownNode(KeyPair.random().publicKey(), URI.create("http://127.0.0.1:9002/"))); return knownNodes; } catch (Exception e) { fail(e); @@ -94,9 +93,9 @@ private void addNodesToNetwork(final Collection nodes) { nodes.forEach(node -> { try { final PublicKey publicKey = PublicKey.fromBytes(Base64.decodeBytes(node.getPublicKey())); - final URL nodeUrl = new URL(node.getNodeUrl()); - networkNodes.addNode(Collections.singletonList(publicKey), nodeUrl); - } catch (MalformedURLException e) { + final URI nodeUri = URI.create(node.getNodeURI()); + networkNodes.addNode(Collections.singletonMap(publicKey, nodeUri).entrySet()); + } catch (IllegalArgumentException e) { fail(e); } }); diff --git a/src/test/java/net/consensys/orion/network/CaOrTofuNodeClientTest.java b/src/test/java/net/consensys/orion/network/CaOrTofuNodeClientTest.java index 75f42649..684b91b1 100644 --- a/src/test/java/net/consensys/orion/network/CaOrTofuNodeClientTest.java +++ b/src/test/java/net/consensys/orion/network/CaOrTofuNodeClientTest.java @@ -23,7 +23,7 @@ import net.consensys.orion.http.server.HttpContentType; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -75,7 +75,8 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { Files.write(knownServersFile, Collections.singletonList("#First line")); final Router dummyRouter = Router.router(vertx); - final ConcurrentNetworkNodes payload = new ConcurrentNetworkNodes(new URL("http://www.example.com")); + final ReadOnlyNetworkNodes payload = + new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyList(), Collections.emptyMap()); dummyRouter.post("/partyinfo").handler(routingContext -> { routingContext.response().end(Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, payload))); }); diff --git a/src/test/java/net/consensys/orion/network/CertificateAuthorityNodeClientTest.java b/src/test/java/net/consensys/orion/network/CertificateAuthorityNodeClientTest.java index 97c7d019..d2201b09 100644 --- a/src/test/java/net/consensys/orion/network/CertificateAuthorityNodeClientTest.java +++ b/src/test/java/net/consensys/orion/network/CertificateAuthorityNodeClientTest.java @@ -24,7 +24,7 @@ import net.consensys.orion.http.server.HttpContentType; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; @@ -72,7 +72,8 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { Files.write(knownServersFile, Collections.singletonList("#First line")); final Router dummyRouter = Router.router(vertx); - final ConcurrentNetworkNodes payload = new ConcurrentNetworkNodes(new URL("http://www.example.com")); + final ReadOnlyNetworkNodes payload = + new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyList(), Collections.emptyMap()); dummyRouter.post("/partyinfo").handler(routingContext -> { routingContext.response().end(Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, payload))); }); diff --git a/src/test/java/net/consensys/orion/network/ConcurrentNetworkNodesTest.java b/src/test/java/net/consensys/orion/network/ConcurrentNetworkNodesTest.java deleted file mode 100644 index 98bb179d..00000000 --- a/src/test/java/net/consensys/orion/network/ConcurrentNetworkNodesTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2018 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ -package net.consensys.orion.network; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import net.consensys.orion.http.server.HttpContentType; -import net.consensys.orion.utils.Serializer; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.apache.tuweni.crypto.sodium.Box; -import org.junit.jupiter.api.Test; - -class ConcurrentNetworkNodesTest { - - @Test - void roundTripSerialization() throws MalformedURLException { - final URL u = new URL("http://nowhere:9090/"); - final List urls = Collections.singletonList(u); - final Map pks = Collections.singletonMap(Box.KeyPair.random().publicKey(), u); - final ConcurrentNetworkNodes nodes = new ConcurrentNetworkNodes(new URL("http://some.server:8080/"), urls, pks); - byte[] bytes = Serializer.serialize(HttpContentType.JSON, nodes); - assertEquals(nodes, Serializer.deserialize(HttpContentType.JSON, ConcurrentNetworkNodes.class, bytes)); - bytes = Serializer.serialize(HttpContentType.CBOR, nodes); - assertEquals(nodes, Serializer.deserialize(HttpContentType.CBOR, ConcurrentNetworkNodes.class, bytes)); - } -} diff --git a/src/test/java/net/consensys/orion/network/InsecureNodeClientTest.java b/src/test/java/net/consensys/orion/network/InsecureNodeClientTest.java index d7a92ef0..d203a3cf 100644 --- a/src/test/java/net/consensys/orion/network/InsecureNodeClientTest.java +++ b/src/test/java/net/consensys/orion/network/InsecureNodeClientTest.java @@ -22,7 +22,7 @@ import net.consensys.orion.http.server.HttpContentType; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -74,7 +74,8 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { client = NodeHttpClientBuilder.build(vertx, config, 100); final Router dummyRouter = Router.router(vertx); - final ConcurrentNetworkNodes payload = new ConcurrentNetworkNodes(new URL("http://www.example.com")); + final ReadOnlyNetworkNodes payload = + new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyList(), Collections.emptyMap()); dummyRouter.post("/partyinfo").handler(routingContext -> { routingContext.response().end(Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, payload))); }); diff --git a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java index 69f0b0cc..af5a88fd 100644 --- a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java +++ b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java @@ -21,9 +21,12 @@ import net.consensys.orion.helpers.FakePeer; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.time.Instant; import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import io.vertx.core.Verticle; import io.vertx.core.Vertx; @@ -33,20 +36,24 @@ import org.apache.tuweni.concurrent.AsyncCompletion; import org.apache.tuweni.concurrent.CompletableAsyncCompletion; import org.apache.tuweni.crypto.sodium.Box; +import org.apache.tuweni.kv.KeyValueStore; +import org.apache.tuweni.kv.MapKeyValueStore; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class NetworkDiscoveryTest { private Vertx vertx; - private ConcurrentNetworkNodes networkNodes; + private PersistentNetworkNodes networkNodes; private Config config; + private KeyValueStore store; @BeforeEach void setUp() throws Exception { vertx = Vertx.vertx(); - networkNodes = new ConcurrentNetworkNodes(new URL("http://localhost1234/")); - config = Config.load("tls='off'"); + config = Config.load("tls='off'\nnodeurl=\"http://localhost:11000\""); + store = MapKeyValueStore.open(new ConcurrentHashMap<>()); + networkNodes = new PersistentNetworkNodes(config, new Box.PublicKey[] {Box.KeyPair.random().publicKey()}, store); } @AfterEach @@ -78,7 +85,7 @@ void networkDiscoveryWithUnresponsivePeer() throws Exception { // add peers final FakePeer fakePeer = new FakePeer(new MockResponse().setSocketPolicy(SocketPolicy.NO_RESPONSE), Box.KeyPair.random().publicKey()); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); // start network discovery final NetworkDiscovery networkDiscovery = new NetworkDiscovery(networkNodes, config, 50, 100); @@ -88,7 +95,7 @@ void networkDiscoveryWithUnresponsivePeer() throws Exception { assertEquals(1, networkDiscovery.discoverers().size()); // ensure the discoverer match our peer URL - final NetworkDiscovery.Discoverer discoverer = networkDiscovery.discoverers().get(fakePeer.getURL().toString()); + final NetworkDiscovery.Discoverer discoverer = networkDiscovery.discoverers().get(fakePeer.getURI()); assertNotNull(discoverer); Thread.sleep(3 * (discoverer.currentRefreshDelay + 200)); @@ -101,8 +108,9 @@ void networkDiscoveryWithUnresponsivePeer() throws Exception { @Test void networkDiscoveryWithMerge() throws Exception { // empty memory nodes, lets' say one peer is alone in his network - final byte[] unknownPeerNetworkNodes = - Serializer.serialize(CBOR, new ConcurrentNetworkNodes(new URL("http://localhost/"))); + final byte[] unknownPeerNetworkNodes = Serializer.serialize( + CBOR, + new ReadOnlyNetworkNodes(URI.create("http://localhost/"), Collections.emptyList(), Collections.emptyMap())); final Buffer unknownPeerBody = new Buffer(); unknownPeerBody.write(unknownPeerNetworkNodes); // create a peer that's not in our current network nodes @@ -110,16 +118,18 @@ void networkDiscoveryWithMerge() throws Exception { new FakePeer(new MockResponse().setBody(unknownPeerBody), Box.KeyPair.random().publicKey()); // create a peer that we know, and that knows the lonely unknown peer. - final ConcurrentNetworkNodes knownPeerNetworkNodes = new ConcurrentNetworkNodes(new URL("http://localhost/")); - knownPeerNetworkNodes.addNode(Collections.singletonList(unknownPeer.publicKey), unknownPeer.getURL()); + final ReadOnlyNetworkNodes knownPeerNetworkNodes = new ReadOnlyNetworkNodes( + URI.create("http://www.example.com"), + Collections.singletonList(unknownPeer.getURI()), + Collections.singletonMap(unknownPeer.publicKey, unknownPeer.getURI())); final Buffer knownPeerBody = new Buffer(); knownPeerBody.write(Serializer.serialize(CBOR, knownPeerNetworkNodes)); final FakePeer knownPeer = new FakePeer(new MockResponse().setBody(knownPeerBody), Box.KeyPair.random().publicKey()); // we know this peer, add it to our network nodes - networkNodes.addNode(Collections.singletonList(knownPeer.publicKey), knownPeer.getURL()); - + boolean added =networkNodes.addNode(Collections.singletonMap(knownPeer.publicKey, knownPeer.getURI()).entrySet()); +assertTrue(added); // start network discovery final Instant discoveryStart = Instant.now(); final NetworkDiscovery networkDiscovery = new NetworkDiscovery(networkNodes, config, 500, 500); @@ -129,8 +139,7 @@ void networkDiscoveryWithMerge() throws Exception { assertEquals(1, networkDiscovery.discoverers().size()); // ensure the discoverer match our peer URL - final NetworkDiscovery.Discoverer knownPeerDiscoverer = - networkDiscovery.discoverers().get(knownPeer.getURL().toString()); + final NetworkDiscovery.Discoverer knownPeerDiscoverer = networkDiscovery.discoverers().get(knownPeer.getURI()); assertNotNull(knownPeerDiscoverer); Thread.sleep(3 * (knownPeerDiscoverer.currentRefreshDelay + 1000)); @@ -142,12 +151,19 @@ void networkDiscoveryWithMerge() throws Exception { assertTrue(knownPeerDiscoverer.attempts >= 2); // ensure we now know unknownPeer - assertEquals(2, networkNodes.nodePKs().size()); - assertEquals(unknownPeer.getURL(), networkNodes.nodePKs().get(unknownPeer.publicKey)); + int size = 0; + Iterator> iter = networkNodes.nodePKs().iterator(); + while (iter.hasNext()) { + size++; + iter.next(); + } + assertEquals(3, size); + assertEquals(unknownPeer.getURI(), networkNodes.uriForRecipient(unknownPeer.publicKey)); + + assertEquals(2, networkDiscovery.discoverers().size()); // ensure unknown peer discoverer is set and being called - final NetworkDiscovery.Discoverer unknownPeerDiscoverer = - networkDiscovery.discoverers().get(unknownPeer.getURL().toString()); + final NetworkDiscovery.Discoverer unknownPeerDiscoverer = networkDiscovery.discoverers().get(unknownPeer.getURI()); assertNotNull(unknownPeerDiscoverer); assertTrue(unknownPeerDiscoverer.lastUpdate.isAfter(discoveryStart)); diff --git a/src/test/java/net/consensys/orion/network/TofuNodeClientTest.java b/src/test/java/net/consensys/orion/network/TofuNodeClientTest.java index c749d4a6..f34c6c41 100644 --- a/src/test/java/net/consensys/orion/network/TofuNodeClientTest.java +++ b/src/test/java/net/consensys/orion/network/TofuNodeClientTest.java @@ -25,7 +25,7 @@ import net.consensys.orion.http.server.HttpContentType; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -81,7 +81,8 @@ void setUp() throws Exception { Files.write(knownServersFile, Collections.singletonList("#First line")); final Router dummyRouter = Router.router(vertx); - final ConcurrentNetworkNodes payload = new ConcurrentNetworkNodes(new URL("http://www.example.com")); + final ReadOnlyNetworkNodes payload = + new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyList(), Collections.emptyMap()); dummyRouter.post("/partyinfo").handler(routingContext -> { routingContext.response().end(Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, payload))); }); diff --git a/src/test/java/net/consensys/orion/network/WhitelistNodeClientTest.java b/src/test/java/net/consensys/orion/network/WhitelistNodeClientTest.java index 91167311..5797b42f 100644 --- a/src/test/java/net/consensys/orion/network/WhitelistNodeClientTest.java +++ b/src/test/java/net/consensys/orion/network/WhitelistNodeClientTest.java @@ -24,11 +24,12 @@ import net.consensys.orion.http.server.HttpContentType; import net.consensys.orion.utils.Serializer; -import java.net.URL; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Collections; import java.util.concurrent.CompletionException; import javax.net.ssl.SSLException; @@ -81,7 +82,8 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { client = NodeHttpClientBuilder.build(vertx, config, 100); - final ConcurrentNetworkNodes payload = new ConcurrentNetworkNodes(new URL("http://www.example.com")); + final ReadOnlyNetworkNodes payload = + new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyList(), Collections.emptyMap()); dummyRouter.post("/partyinfo").handler(routingContext -> { routingContext.response().end(Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, payload))); }); diff --git a/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java b/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java index 43231788..042ee33d 100644 --- a/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java +++ b/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import net.consensys.orion.cmd.Orion; import net.consensys.orion.config.Config; import net.consensys.orion.enclave.Enclave; import net.consensys.orion.enclave.EncryptedPayload; @@ -38,7 +39,7 @@ import net.consensys.orion.helpers.StubEnclave; import net.consensys.orion.http.handler.send.SendRequest; import net.consensys.orion.http.handler.send.SendResponse; -import net.consensys.orion.network.ConcurrentNetworkNodes; +import net.consensys.orion.network.PersistentNetworkNodes; import net.consensys.orion.storage.EncryptedPayloadStorage; import net.consensys.orion.storage.PrivacyGroupStorage; import net.consensys.orion.storage.QueryPrivacyGroupStorage; @@ -59,6 +60,7 @@ import io.vertx.junit5.VertxTestContext; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.RecordedRequest; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.crypto.sodium.Box; import org.apache.tuweni.crypto.sodium.Box.PublicKey; import org.apache.tuweni.junit.TempDirectory; @@ -80,14 +82,14 @@ class DistributePayloadManagerTest { private Storage payloadStorage; private Storage queryPrivacyGroupStorage; private Storage privacyGroupStorage; - private ConcurrentNetworkNodes networkNodes; + private PersistentNetworkNodes networkNodes; private DistributePayloadManager distributePayloadManager; static { Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider()); } - private KeyValueStore storage; + private KeyValueStore storage; @BeforeEach public void beforeEach(@TempDirectory final Path tempDir) throws Exception { @@ -102,7 +104,7 @@ public void beforeEach(@TempDirectory final Path tempDir) throws Exception { payloadStorage = new EncryptedPayloadStorage(storage, keyBuilder); privacyGroupStorage = new PrivacyGroupStorage(storage, enclave); queryPrivacyGroupStorage = new QueryPrivacyGroupStorage(storage, enclave); - networkNodes = new ConcurrentNetworkNodes(config, enclave.nodeKeys()); + networkNodes = new PersistentNetworkNodes(config, enclave.nodeKeys(), Orion.wrap(storage)); distributePayloadManager = new DistributePayloadManager( enclave, @@ -122,7 +124,7 @@ void tearDown() throws Exception { @Test public void failsWhenBadResponseFromPeer(final VertxTestContext testContext) throws Exception { final FakePeer fakePeer = new FakePeer(new MockResponse().setResponseCode(500), memoryKeyStore); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); final SendRequest request = buildLegacyRequest(Collections.singletonList(fakePeer), "foo".getBytes(UTF_8)); final OrionException expectedException = new OrionException(OrionErrorCode.NODE_PROPAGATING_TO_ALL_PEERS); @@ -136,7 +138,7 @@ public void failsWhenBadResponseFromPeer(final VertxTestContext testContext) thr @Test public void failsWhenBadDigestFromPeer(final VertxTestContext testContext) throws Exception { final FakePeer fakePeer = new FakePeer(new MockResponse().setBody("not the best digest"), memoryKeyStore); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); final SendRequest request = buildLegacyRequest(Collections.singletonList(fakePeer), "foo".getBytes(UTF_8)); final OrionException expectedException = new OrionException(OrionErrorCode.NODE_PROPAGATING_TO_ALL_PEERS); @@ -207,7 +209,7 @@ public void distributePayloadToSinglePeerUsingLegacyWay(final VertxTestContext t // create fake peer final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(digest), memoryKeyStore); - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); final SendRequest request = buildLegacyRequest(Collections.singletonList(fakePeer), toEncrypt); @@ -235,7 +237,7 @@ public void distributePayloadToMultiplePeerUsingLegacyWay(final VertxTestContext for (int i = 0; i < 5; i++) { final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(digest), memoryKeyStore); // add peer push URL to networkNodes - networkNodes.addNode(Collections.singletonList(fakePeer.publicKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); fakePeers.add(fakePeer); } @@ -267,7 +269,7 @@ public void distributePayloadToSinglePeerUsingPrivacyGroup(final VertxTestContex // create fake peer final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(digest), recipientKey); - networkNodes.addNode(Collections.singletonList(recipientKey), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); final String from = encodeBytes(senderKey.bytesArray()); final String to = encodeBytes(fakePeer.publicKey.bytesArray()); @@ -311,7 +313,7 @@ public void distributePayloadToMultiplePeerUsingPrivacyGroup(final VertxTestCont for (int i = 0; i < numOfPeers; i++) { // create fake peer final FakePeer fakePeer = new FakePeer(new MockResponse().setBody(digest), recipientKeys[i]); - networkNodes.addNode(Collections.singletonList(recipientKeys[i]), fakePeer.getURL()); + networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); fakePeers.add(fakePeer); } diff --git a/src/test/java/net/consensys/orion/storage/EncryptedPayloadStorageTest.java b/src/test/java/net/consensys/orion/storage/EncryptedPayloadStorageTest.java index a5ab438d..308efb8d 100644 --- a/src/test/java/net/consensys/orion/storage/EncryptedPayloadStorageTest.java +++ b/src/test/java/net/consensys/orion/storage/EncryptedPayloadStorageTest.java @@ -34,7 +34,7 @@ class EncryptedPayloadStorageTest { private final Enclave enclave = new SodiumEnclaveStub(); private final StorageKeyBuilder keyBuilder = new Sha512_256StorageKeyBuilder(); private final Storage payloadStorage = - new EncryptedPayloadStorage(new MapKeyValueStore(), keyBuilder); + new EncryptedPayloadStorage(new MapKeyValueStore<>(), keyBuilder); @Test void storeAndRetrieve() throws Exception { diff --git a/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java b/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java index 0dc7cfd1..a4f82388 100644 --- a/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java +++ b/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java @@ -23,7 +23,6 @@ import net.consensys.orion.enclave.sodium.SodiumEnclave; import java.nio.file.Path; -import java.security.Security; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -31,22 +30,23 @@ import java.util.Optional; import java.util.Random; +import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.crypto.sodium.Box; +import org.apache.tuweni.io.Base64; +import org.apache.tuweni.junit.BouncyCastleExtension; import org.apache.tuweni.junit.TempDirectory; import org.apache.tuweni.junit.TempDirectoryExtension; +import org.apache.tuweni.kv.EntityManagerKeyValueStore; import org.apache.tuweni.kv.KeyValueStore; +import org.apache.tuweni.kv.ProxyKeyValueStore; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -@ExtendWith(TempDirectoryExtension.class) +@ExtendWith({TempDirectoryExtension.class, BouncyCastleExtension.class}) class PrivacyGroupStorageTest { private MemoryKeyStore memoryKeyStore; - private KeyValueStore storage; - - static { - Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider()); - } + private KeyValueStore storage; private Enclave enclave; private Storage payloadStorage; @@ -64,7 +64,17 @@ void setup(@TempDirectory final Path tempDir) throws SQLException { st.executeUpdate("create table if not exists store(key char(60), value binary, primary key(key))"); } final JpaEntityManagerProvider jpaEntityManagerProvider = new JpaEntityManagerProvider(jdbcUrl); - storage = new OrionSQLKeyValueStore(jpaEntityManagerProvider); + storage = ProxyKeyValueStore.open( + EntityManagerKeyValueStore.open(jpaEntityManagerProvider::createEntityManager, Store.class, Store::getKey), + Base64::decode, + Base64::encode, + store -> Bytes.concatenate(Base64.decode(store.getKey()), Bytes.wrap(store.getValue())), + value -> { + Store store = new Store(); + store.setKey(Base64.encode(value.slice(0, 32))); + store.setValue(value.slice(32).toArrayUnsafe()); + return store; + }); payloadStorage = new PrivacyGroupStorage(storage, enclave); } From 98b327af22ca97c4d9b12145e6ffef1b2c23bfef Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Sat, 15 Feb 2020 09:26:06 -0800 Subject: [PATCH 02/19] Fix tests --- .../consensys/orion/network/NetworkDiscovery.java | 10 ++++++---- .../orion/network/PersistentNetworkNodes.java | 1 + .../orion/network/NetworkDiscoveryTest.java | 15 ++++++++------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/consensys/orion/network/NetworkDiscovery.java b/src/main/java/net/consensys/orion/network/NetworkDiscovery.java index c30b31a1..bfdb5f23 100644 --- a/src/main/java/net/consensys/orion/network/NetworkDiscovery.java +++ b/src/main/java/net/consensys/orion/network/NetworkDiscovery.java @@ -83,13 +83,15 @@ public void stop() { * when a merge occurs in one of the Discoverer (same event loop) */ private void updateDiscoverers() { + log.trace("Updating discoverers"); // for each peer that we know, we start a Discoverer (on timer) for (final URI nodeUri : nodes.nodeURIs()) { - if (!(nodeUri.equals(nodes.uri()) || discoverers.containsKey(nodeUri))) { - final Discoverer d = new Discoverer(nodeUri, refreshDelayMs, nodes.uri().equals(nodeUri)); - discoverers.put(nodeUri, d); + discoverers.computeIfAbsent(nodeUri, uri -> { + log.trace("New discoverer for {}", uri); + final Discoverer d = new Discoverer(uri, refreshDelayMs, uri.equals(nodes.uri())); d.engageNextTimerTick(); - } + return d; + }); } } diff --git a/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java b/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java index 0040d94f..a729ec1c 100644 --- a/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java +++ b/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java @@ -74,6 +74,7 @@ public void setNodeUrl(final URI uri, final Box.PublicKey[] publicKeys) { * @param nodesPks List of PublicKeys of new node */ public boolean addNode(final Iterable> nodesPks) { + logger.trace("addNode called"); AtomicBoolean changed = new AtomicBoolean(false); List completions = new ArrayList<>(); nodesPks.forEach(entry -> { diff --git a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java index af5a88fd..8f5c6e22 100644 --- a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java +++ b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java @@ -92,7 +92,7 @@ void networkDiscoveryWithUnresponsivePeer() throws Exception { deployVerticle(networkDiscovery).join(); // assert the discoverer started - assertEquals(1, networkDiscovery.discoverers().size()); + assertEquals(2, networkDiscovery.discoverers().size()); // ensure the discoverer match our peer URL final NetworkDiscovery.Discoverer discoverer = networkDiscovery.discoverers().get(fakePeer.getURI()); @@ -128,21 +128,21 @@ void networkDiscoveryWithMerge() throws Exception { new FakePeer(new MockResponse().setBody(knownPeerBody), Box.KeyPair.random().publicKey()); // we know this peer, add it to our network nodes - boolean added =networkNodes.addNode(Collections.singletonMap(knownPeer.publicKey, knownPeer.getURI()).entrySet()); -assertTrue(added); + boolean added = networkNodes.addNode(Collections.singletonMap(knownPeer.publicKey, knownPeer.getURI()).entrySet()); + assertTrue(added); // start network discovery final Instant discoveryStart = Instant.now(); final NetworkDiscovery networkDiscovery = new NetworkDiscovery(networkNodes, config, 500, 500); deployVerticle(networkDiscovery).join(); // assert the discoverer started, we should only have 1 discoverer for knownPeer - assertEquals(1, networkDiscovery.discoverers().size()); + assertEquals(2, networkDiscovery.discoverers().size()); // ensure the discoverer match our peer URL final NetworkDiscovery.Discoverer knownPeerDiscoverer = networkDiscovery.discoverers().get(knownPeer.getURI()); assertNotNull(knownPeerDiscoverer); - Thread.sleep(3 * (knownPeerDiscoverer.currentRefreshDelay + 1000)); + Thread.sleep(3 * (knownPeerDiscoverer.currentRefreshDelay + 2000)); // ensure knownPeer responded and that his party info was called at least twice assertTrue( @@ -155,12 +155,13 @@ void networkDiscoveryWithMerge() throws Exception { Iterator> iter = networkNodes.nodePKs().iterator(); while (iter.hasNext()) { size++; - iter.next(); + Map.Entry entry = iter.next(); + System.out.println(entry.getValue()); } assertEquals(3, size); assertEquals(unknownPeer.getURI(), networkNodes.uriForRecipient(unknownPeer.publicKey)); - assertEquals(2, networkDiscovery.discoverers().size()); + assertEquals(3, networkDiscovery.discoverers().size()); // ensure unknown peer discoverer is set and being called final NetworkDiscovery.Discoverer unknownPeerDiscoverer = networkDiscovery.discoverers().get(unknownPeer.getURI()); From 6446d8a69a19dc9656309ce96d00444ffb6a31e3 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Sun, 16 Feb 2020 00:49:48 -0800 Subject: [PATCH 03/19] Fix a couple of tests --- .../java/net/consensys/orion/cmd/Orion.java | 10 +++++----- .../orion/storage/PrivacyGroupStorage.java | 1 + .../orion/network/NetworkDiscoveryTest.java | 18 ++++++++---------- .../orion/storage/PrivacyGroupStorageTest.java | 8 ++++---- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/main/java/net/consensys/orion/cmd/Orion.java b/src/main/java/net/consensys/orion/cmd/Orion.java index eff897f3..e1d76e1e 100644 --- a/src/main/java/net/consensys/orion/cmd/Orion.java +++ b/src/main/java/net/consensys/orion/cmd/Orion.java @@ -543,7 +543,7 @@ public static KeyValueStore wrap(KeyValueStore .open(store, Box.PublicKey::fromBytes, Box.PublicKey::bytes, Orion::bytesToURI, Orion::uriToBytes); } - private static Bytes uriToBytes(URI uri) { + private static Bytes uriToBytes(Box.PublicKey key, URI uri) { return Bytes.wrap(uri.toString().getBytes(StandardCharsets.UTF_8)); } @@ -578,11 +578,11 @@ private KeyValueStore createStorage(final String storage, final Pa EntityManagerKeyValueStore.open(jpaEntityManagerProvider::createEntityManager, Store.class, Store::getKey), Base64::decode, Base64::encode, - store -> Bytes.concatenate(Base64.decode(store.getKey()), Bytes.wrap(store.getValue())), - value -> { + store -> Bytes.wrap(store.getValue()), + (Bytes key, Bytes value) -> { Store store = new Store(); - store.setKey(Base64.encode(value.slice(0, 32))); - store.setValue(value.slice(32).toArrayUnsafe()); + store.setKey(Base64.encode(key)); + store.setValue(value.toArrayUnsafe()); return store; }); } else { diff --git a/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java b/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java index 3c40c849..b8252e80 100644 --- a/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java +++ b/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java @@ -28,6 +28,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.concurrent.AsyncResult; import org.apache.tuweni.crypto.sodium.Box; +import org.apache.tuweni.io.Base64; import org.apache.tuweni.kv.KeyValueStore; diff --git a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java index 8f5c6e22..3847ca2a 100644 --- a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java +++ b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java @@ -51,7 +51,7 @@ class NetworkDiscoveryTest { @BeforeEach void setUp() throws Exception { vertx = Vertx.vertx(); - config = Config.load("tls='off'\nnodeurl=\"http://localhost:11000\""); + config = Config.load("tls='off'"); store = MapKeyValueStore.open(new ConcurrentHashMap<>()); networkNodes = new PersistentNetworkNodes(config, new Box.PublicKey[] {Box.KeyPair.random().publicKey()}, store); } @@ -88,21 +88,21 @@ void networkDiscoveryWithUnresponsivePeer() throws Exception { networkNodes.addNode(Collections.singletonMap(fakePeer.publicKey, fakePeer.getURI()).entrySet()); // start network discovery - final NetworkDiscovery networkDiscovery = new NetworkDiscovery(networkNodes, config, 50, 100); + final NetworkDiscovery networkDiscovery = new NetworkDiscovery(networkNodes, config, 50, 10); deployVerticle(networkDiscovery).join(); // assert the discoverer started - assertEquals(2, networkDiscovery.discoverers().size()); + assertEquals(1, networkDiscovery.discoverers().size()); // ensure the discoverer match our peer URL final NetworkDiscovery.Discoverer discoverer = networkDiscovery.discoverers().get(fakePeer.getURI()); assertNotNull(discoverer); - Thread.sleep(3 * (discoverer.currentRefreshDelay + 200)); + Thread.sleep(3 * (discoverer.currentRefreshDelay + 1000)); // ensure we didn't do any update, and we tried at least 2 times assertEquals(Instant.MIN, discoverer.lastUpdate); - assertTrue(discoverer.attempts >= 2); + assertTrue(discoverer.attempts >= 2, "Tried " + discoverer.attempts + " times"); } @Test @@ -136,7 +136,7 @@ void networkDiscoveryWithMerge() throws Exception { deployVerticle(networkDiscovery).join(); // assert the discoverer started, we should only have 1 discoverer for knownPeer - assertEquals(2, networkDiscovery.discoverers().size()); + assertEquals(1, networkDiscovery.discoverers().size()); // ensure the discoverer match our peer URL final NetworkDiscovery.Discoverer knownPeerDiscoverer = networkDiscovery.discoverers().get(knownPeer.getURI()); @@ -155,13 +155,11 @@ void networkDiscoveryWithMerge() throws Exception { Iterator> iter = networkNodes.nodePKs().iterator(); while (iter.hasNext()) { size++; - Map.Entry entry = iter.next(); - System.out.println(entry.getValue()); } - assertEquals(3, size); + assertEquals(2, size); assertEquals(unknownPeer.getURI(), networkNodes.uriForRecipient(unknownPeer.publicKey)); - assertEquals(3, networkDiscovery.discoverers().size()); + assertEquals(2, networkDiscovery.discoverers().size()); // ensure unknown peer discoverer is set and being called final NetworkDiscovery.Discoverer unknownPeerDiscoverer = networkDiscovery.discoverers().get(unknownPeer.getURI()); diff --git a/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java b/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java index a4f82388..57df5b60 100644 --- a/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java +++ b/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java @@ -68,11 +68,11 @@ void setup(@TempDirectory final Path tempDir) throws SQLException { EntityManagerKeyValueStore.open(jpaEntityManagerProvider::createEntityManager, Store.class, Store::getKey), Base64::decode, Base64::encode, - store -> Bytes.concatenate(Base64.decode(store.getKey()), Bytes.wrap(store.getValue())), - value -> { + store -> Bytes.wrap(store.getValue()), + (key, value) -> { Store store = new Store(); - store.setKey(Base64.encode(value.slice(0, 32))); - store.setValue(value.slice(32).toArrayUnsafe()); + store.setKey(Base64.encode(key)); + store.setValue(value.toArrayUnsafe()); return store; }); payloadStorage = new PrivacyGroupStorage(storage, enclave); From 5e263fc60b4914d0724c77e0e4b16fdfb5bf4ed6 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Sun, 16 Feb 2020 00:50:03 -0800 Subject: [PATCH 04/19] Fix acceptance tests compilation --- .../consensys/orion/acceptance/NodeUtils.java | 10 +++---- .../acceptance/send/SingleNodeSendTest.java | 1 - .../receive/DualNodesSendReceiveTest.java | 29 ++++++++++++++----- .../DualNodesPrivacyGroupsTest.java | 20 +++++++++---- ...NodesSendReceiveUsingPrivacyGroupTest.java | 22 +++++++++----- 5 files changed, 55 insertions(+), 27 deletions(-) diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java b/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java index 6bf4328b..bdb52751 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java @@ -23,7 +23,7 @@ import java.io.IOException; import java.net.ServerSocket; -import java.net.URL; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -46,16 +46,16 @@ public static String joinPathsAsTomlListEntry(final Path... paths) { return builder.toString(); } - private static HttpUrl buildUrl(final String host, final int port) { + private static HttpUrl buildUri(final String host, final int port) { return new HttpUrl.Builder().scheme("http").host(host).port(port).build(); } public static String urlString(final String host, final int port) { - return buildUrl(host, port).toString(); + return buildUri(host, port).toString(); } - public static URL url(final String host, final int port) { - return buildUrl(host, port).url(); + public static URI uri(final String host, final int port) { + return buildUri(host, port).uri(); } public static Config nodeConfig( diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java index bd4961d0..cba0ac3a 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java @@ -48,7 +48,6 @@ class SingleNodeSendTest { private static final String PK_2_B_64 = "Ko2bVqD+nNlNYL5EE7y3IdOnviftjiizpjRt+HTuFBs="; private static final String PK_CORRUPTED = "A1aVtMxLCUHmBVHXoZzzBgPbW/wj5axDpW9X8l91SGoAAA="; private static final String PK_MISSING_PEER = "A1aVtMxLCUlNYL5EE7y3IdOnviftjiizpjRt+HTuFBs="; - private static final String HOST_NAME = "127.0.0.1"; private static Config config; diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java index 02342363..196057b5 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java @@ -30,16 +30,20 @@ import net.consensys.orion.config.Config; import net.consensys.orion.http.server.HttpContentType; import net.consensys.orion.network.PersistentNetworkNodes; +import net.consensys.orion.network.ReadOnlyNetworkNodes; import net.consensys.orion.utils.Serializer; import java.io.IOException; +import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -56,6 +60,7 @@ import org.apache.tuweni.crypto.sodium.Box; import org.apache.tuweni.junit.TempDirectory; import org.apache.tuweni.junit.TempDirectoryExtension; +import org.apache.tuweni.kv.MapKeyValueStore; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -129,10 +134,17 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { final Box.PublicKey pk3 = Box.PublicKey.fromBytes(decodeBytes(pubKeyStrings.get(2))); final Box.PublicKey pk4 = Box.PublicKey.fromBytes(decodeBytes(pubKeyStrings.get(3))); final Box.PublicKey pk5 = Box.PublicKey.fromBytes(decodeBytes(pubKeyStrings.get(4))); - networkNodes = new PersistentNetworkNodes(NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); + networkNodes = new PersistentNetworkNodes(firstNodeConfig, new Box.PublicKey[] { } , MapKeyValueStore.open()); + networkNodes.setNodeUrl(NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort()), new Box.PublicKey[0]); + Map pks = new HashMap<>(); + pks.put(pk1, NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort())); + pks.put(pk2, NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort())); + pks.put(pk3, NodeUtils.uri("127.0.0.1", secondOrionLauncher.nodePort())); + pks.put(pk4, NodeUtils.uri("127.0.0.1", secondOrionLauncher.nodePort())); + pks.put(pk5, NodeUtils.uri("127.0.0.1", secondOrionLauncher.nodePort())); + + networkNodes.addNode(pks.entrySet()); - networkNodes.addNode(Arrays.asList(pk1, pk2), NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); - networkNodes.addNode(Arrays.asList(pk3, pk4, pk5), NodeUtils.url("127.0.0.1", secondOrionLauncher.nodePort())); // prepare /partyinfo payload (our known peers) final RequestBody partyInfoBody = RequestBody.create(MediaType.parse(CBOR.httpHeaderValue), Serializer.serialize(CBOR, networkNodes)); @@ -142,7 +154,7 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { final String firstNodeBaseUrl = NodeUtils.urlString("127.0.0.1", firstOrionLauncher.nodePort()); final Request request = new Request.Builder().post(partyInfoBody).url(firstNodeBaseUrl + "/partyinfo").build(); // first /partyinfo call may just get the one node, so wait until we get at least 2 nodes - await().atMost(10, TimeUnit.SECONDS).until(() -> getPartyInfoResponse(httpClient, request).nodeURLs().size() == 2); + await().atMost(10, TimeUnit.SECONDS).until(() -> getPartyInfoResponse(httpClient, request).nodeURIs().size() == 2); } @@ -163,13 +175,14 @@ private String getStringFromResource(final String resourceFileName) { } } - private PersistentNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) + private ReadOnlyNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) throws Exception { final Response resp = httpClient.newCall(request).execute(); - assertEquals(200, resp.code()); + System.out.println(resp.body().string()); + assertEquals("Received " + resp.code(), 200, resp.code()); - final PersistentNetworkNodes partyInfoResponse = - Serializer.deserialize(HttpContentType.CBOR, PersistentNetworkNodes.class, resp.body().bytes()); + final ReadOnlyNetworkNodes partyInfoResponse = + Serializer.deserialize(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, resp.body().bytes()); return partyInfoResponse; } diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java index 66b5b9bd..984baa4c 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java @@ -36,15 +36,19 @@ import net.consensys.orion.http.handler.privacy.PrivacyGroup; import net.consensys.orion.http.server.HttpContentType; import net.consensys.orion.network.PersistentNetworkNodes; +import net.consensys.orion.network.ReadOnlyNetworkNodes; import net.consensys.orion.utils.Serializer; +import java.net.URI; import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -58,6 +62,7 @@ import okhttp3.Response; import org.apache.tuweni.crypto.sodium.Box; import org.apache.tuweni.junit.TempDirectoryExtension; +import org.apache.tuweni.kv.MapKeyValueStore; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -125,10 +130,13 @@ void setUpDualNodes(@TempDir final Path tempDir) throws Exception { final Box.PublicKey pk1 = Box.PublicKey.fromBytes(decodeBytes(PK_1_B_64)); final Box.PublicKey pk2 = Box.PublicKey.fromBytes(decodeBytes(PK_2_B_64)); final PersistentNetworkNodes networkNodes = - new PersistentNetworkNodes(NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); + new PersistentNetworkNodes(firstNodeConfig, new Box.PublicKey[] { } , MapKeyValueStore.open()); + networkNodes.setNodeUrl(NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort()), new Box.PublicKey[0]); + Map pks = new HashMap<>(); + pks.put(pk1, NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort())); + pks.put(pk2, NodeUtils.uri("127.0.0.1", secondOrionLauncher.nodePort())); - networkNodes.addNode(Collections.singletonList(pk1), NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); - networkNodes.addNode(Collections.singletonList(pk2), NodeUtils.url("127.0.0.1", secondOrionLauncher.nodePort())); + networkNodes.addNode(pks.entrySet()); // prepare /partyinfo payload (our known peers) final RequestBody partyInfoBody = RequestBody.create(MediaType.parse(CBOR.httpHeaderValue), Serializer.serialize(CBOR, networkNodes)); @@ -138,16 +146,16 @@ void setUpDualNodes(@TempDir final Path tempDir) throws Exception { final String firstNodeBaseUrl = NodeUtils.urlString("127.0.0.1", firstOrionLauncher.nodePort()); final Request request = new Request.Builder().post(partyInfoBody).url(firstNodeBaseUrl + "/partyinfo").build(); // first /partyinfo call may just get the one node, so wait until we get at least 2 nodes - await().atMost(5, TimeUnit.SECONDS).until(() -> getPartyInfoResponse(httpClient, request).nodeURLs().size() == 2); + await().atMost(5, TimeUnit.SECONDS).until(() -> getPartyInfoResponse(httpClient, request).nodeURIs().size() == 2); } - private PersistentNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) + private ReadOnlyNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) throws Exception { final Response resp = httpClient.newCall(request).execute(); assertEquals(200, resp.code()); - return Serializer.deserialize(HttpContentType.CBOR, PersistentNetworkNodes.class, resp.body().bytes()); + return Serializer.deserialize(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, resp.body().bytes()); } @AfterEach diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java index 51c49af3..b97b2984 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java @@ -33,13 +33,17 @@ import net.consensys.orion.http.handler.receive.ReceiveResponse; import net.consensys.orion.http.server.HttpContentType; import net.consensys.orion.network.PersistentNetworkNodes; +import net.consensys.orion.network.ReadOnlyNetworkNodes; import net.consensys.orion.utils.Serializer; +import java.net.URI; import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.TimeUnit; import io.vertx.core.Vertx; @@ -52,6 +56,7 @@ import org.apache.tuweni.crypto.sodium.Box; import org.apache.tuweni.junit.TempDirectory; import org.apache.tuweni.junit.TempDirectoryExtension; +import org.apache.tuweni.kv.MapKeyValueStore; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -121,10 +126,13 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { secondHttpClient = vertx.createHttpClient(); final Box.PublicKey pk1 = Box.PublicKey.fromBytes(decodeBytes(PK_1_B_64)); final Box.PublicKey pk2 = Box.PublicKey.fromBytes(decodeBytes(PK_2_B_64)); - networkNodes = new PersistentNetworkNodes(NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); + networkNodes = new PersistentNetworkNodes(firstNodeConfig, new Box.PublicKey[] { } , MapKeyValueStore.open()); + networkNodes.setNodeUrl(NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort()), new Box.PublicKey[0]); + Map pks = new HashMap<>(); + pks.put(pk1, NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort())); + pks.put(pk2, NodeUtils.uri("127.0.0.1", secondOrionLauncher.nodePort())); - networkNodes.addNode(Collections.singletonList(pk1), NodeUtils.url("127.0.0.1", firstOrionLauncher.nodePort())); - networkNodes.addNode(Collections.singletonList(pk2), NodeUtils.url("127.0.0.1", secondOrionLauncher.nodePort())); + networkNodes.addNode(pks.entrySet()); // prepare /partyinfo payload (our known peers) final RequestBody partyInfoBody = RequestBody.create(MediaType.parse(CBOR.httpHeaderValue), Serializer.serialize(CBOR, networkNodes)); @@ -134,17 +142,17 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { final String firstNodeBaseUrl = NodeUtils.urlString("127.0.0.1", firstOrionLauncher.nodePort()); final Request request = new Request.Builder().post(partyInfoBody).url(firstNodeBaseUrl + "/partyinfo").build(); // first /partyinfo call may just get the one node, so wait until we get at least 2 nodes - await().atMost(5, TimeUnit.SECONDS).until(() -> getPartyInfoResponse(httpClient, request).nodeURLs().size() == 2); + await().atMost(5, TimeUnit.SECONDS).until(() -> getPartyInfoResponse(httpClient, request).nodeURIs().size() == 2); } - private PersistentNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) + private ReadOnlyNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) throws Exception { final Response resp = httpClient.newCall(request).execute(); assertEquals(200, resp.code()); - final PersistentNetworkNodes partyInfoResponse = - Serializer.deserialize(HttpContentType.CBOR, PersistentNetworkNodes.class, resp.body().bytes()); + final ReadOnlyNetworkNodes partyInfoResponse = + Serializer.deserialize(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, resp.body().bytes()); return partyInfoResponse; } From 5948a869da0f1cfb8d2f6b0f3cdbd38809f02dc1 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Sun, 16 Feb 2020 10:57:21 -0800 Subject: [PATCH 05/19] Test fixes --- .../send/receive/DualNodesSendReceiveTest.java | 3 +-- .../DualNodesPrivacyGroupsTest.java | 3 +-- ...alNodesSendReceiveUsingPrivacyGroupTest.java | 3 +-- .../orion/network/ReadOnlyNetworkNodes.java | 17 ++++++++++------- .../orion/storage/PrivacyGroupStorage.java | 1 - .../http/handler/PartyInfoHandlerTest.java | 1 - .../orion/network/CaOrTofuNodeClientTest.java | 2 +- .../CertificateAuthorityNodeClientTest.java | 2 +- .../orion/network/InsecureNodeClientTest.java | 2 +- .../orion/network/NetworkDiscoveryTest.java | 11 ++++++----- .../orion/network/TofuNodeClientTest.java | 2 +- .../orion/network/WhitelistNodeClientTest.java | 2 +- 12 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java index 196057b5..b5d610ba 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java @@ -40,7 +40,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -134,7 +133,7 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { final Box.PublicKey pk3 = Box.PublicKey.fromBytes(decodeBytes(pubKeyStrings.get(2))); final Box.PublicKey pk4 = Box.PublicKey.fromBytes(decodeBytes(pubKeyStrings.get(3))); final Box.PublicKey pk5 = Box.PublicKey.fromBytes(decodeBytes(pubKeyStrings.get(4))); - networkNodes = new PersistentNetworkNodes(firstNodeConfig, new Box.PublicKey[] { } , MapKeyValueStore.open()); + networkNodes = new PersistentNetworkNodes(firstNodeConfig, new Box.PublicKey[] {}, MapKeyValueStore.open()); networkNodes.setNodeUrl(NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort()), new Box.PublicKey[0]); Map pks = new HashMap<>(); pks.put(pk1, NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort())); diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java index 984baa4c..431f648f 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java @@ -45,7 +45,6 @@ import java.sql.DriverManager; import java.sql.Statement; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -130,7 +129,7 @@ void setUpDualNodes(@TempDir final Path tempDir) throws Exception { final Box.PublicKey pk1 = Box.PublicKey.fromBytes(decodeBytes(PK_1_B_64)); final Box.PublicKey pk2 = Box.PublicKey.fromBytes(decodeBytes(PK_2_B_64)); final PersistentNetworkNodes networkNodes = - new PersistentNetworkNodes(firstNodeConfig, new Box.PublicKey[] { } , MapKeyValueStore.open()); + new PersistentNetworkNodes(firstNodeConfig, new Box.PublicKey[] {}, MapKeyValueStore.open()); networkNodes.setNodeUrl(NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort()), new Box.PublicKey[0]); Map pks = new HashMap<>(); pks.put(pk1, NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort())); diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java index b97b2984..e39f7aba 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java @@ -41,7 +41,6 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -126,7 +125,7 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { secondHttpClient = vertx.createHttpClient(); final Box.PublicKey pk1 = Box.PublicKey.fromBytes(decodeBytes(PK_1_B_64)); final Box.PublicKey pk2 = Box.PublicKey.fromBytes(decodeBytes(PK_2_B_64)); - networkNodes = new PersistentNetworkNodes(firstNodeConfig, new Box.PublicKey[] { } , MapKeyValueStore.open()); + networkNodes = new PersistentNetworkNodes(firstNodeConfig, new Box.PublicKey[] {}, MapKeyValueStore.open()); networkNodes.setNodeUrl(NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort()), new Box.PublicKey[0]); Map pks = new HashMap<>(); pks.put(pk1, NodeUtils.uri("127.0.0.1", firstOrionLauncher.nodePort())); diff --git a/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java b/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java index 99cc296f..01637451 100644 --- a/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java +++ b/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java @@ -15,12 +15,13 @@ import net.consensys.orion.enclave.sodium.serialization.PublicKeyMapKeyDeserializer; import java.net.URI; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.CopyOnWriteArrayList; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -30,20 +31,22 @@ public class ReadOnlyNetworkNodes implements NetworkNodes { private final URI uri; - private final CopyOnWriteArrayList nodeURIs; private final Map nodePKs; @JsonCreator public ReadOnlyNetworkNodes( @JsonProperty("uri") final URI uri, - @JsonProperty("nodeURIs") final List nodeURIs, + @JsonProperty("nodeURIs") List nodeURIs, @JsonProperty("nodePKs") @JsonDeserialize( keyUsing = PublicKeyMapKeyDeserializer.class) final Map nodePKs) { this.uri = uri; - this.nodeURIs = new CopyOnWriteArrayList<>(nodeURIs); this.nodePKs = new HashMap<>(nodePKs); } + public ReadOnlyNetworkNodes(URI uri, Map nodePKs) { + this(uri, Collections.emptyList(), nodePKs); + } + @Override public URI uri() { return uri; @@ -51,7 +54,7 @@ public URI uri() { @Override public Collection nodeURIs() { - return nodeURIs; + return new ArrayList<>(nodePKs.values()); } @Override @@ -71,11 +74,11 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; ReadOnlyNetworkNodes that = (ReadOnlyNetworkNodes) o; - return uri.equals(that.uri) && nodeURIs.equals(that.nodeURIs) && nodePKs.equals(that.nodePKs); + return uri.equals(that.uri) && nodePKs.equals(that.nodePKs); } @Override public int hashCode() { - return Objects.hash(uri, nodeURIs, nodePKs); + return Objects.hash(uri, nodePKs); } } diff --git a/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java b/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java index b8252e80..3c40c849 100644 --- a/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java +++ b/src/main/java/net/consensys/orion/storage/PrivacyGroupStorage.java @@ -28,7 +28,6 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.concurrent.AsyncResult; import org.apache.tuweni.crypto.sodium.Box; -import org.apache.tuweni.io.Base64; import org.apache.tuweni.kv.KeyValueStore; diff --git a/src/test/java/net/consensys/orion/http/handler/PartyInfoHandlerTest.java b/src/test/java/net/consensys/orion/http/handler/PartyInfoHandlerTest.java index c81539aa..ee23d364 100644 --- a/src/test/java/net/consensys/orion/http/handler/PartyInfoHandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/PartyInfoHandlerTest.java @@ -62,7 +62,6 @@ void successfulProcessingOfRequest() throws Exception { void roundTripSerialization() throws Exception { final ReadOnlyNetworkNodes networkNodes = new ReadOnlyNetworkNodes( URI.create("http://localhost:1234/"), - Collections.singletonList(URI.create("http://localhost/")), Collections.singletonMap(Box.KeyPair.random().publicKey(), URI.create("http://localhost/"))); assertEquals(networkNodes, Serializer.roundTrip(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, networkNodes)); assertEquals(networkNodes, Serializer.roundTrip(HttpContentType.JSON, ReadOnlyNetworkNodes.class, networkNodes)); diff --git a/src/test/java/net/consensys/orion/network/CaOrTofuNodeClientTest.java b/src/test/java/net/consensys/orion/network/CaOrTofuNodeClientTest.java index 684b91b1..719b5a6d 100644 --- a/src/test/java/net/consensys/orion/network/CaOrTofuNodeClientTest.java +++ b/src/test/java/net/consensys/orion/network/CaOrTofuNodeClientTest.java @@ -76,7 +76,7 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { final Router dummyRouter = Router.router(vertx); final ReadOnlyNetworkNodes payload = - new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyList(), Collections.emptyMap()); + new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyMap()); dummyRouter.post("/partyinfo").handler(routingContext -> { routingContext.response().end(Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, payload))); }); diff --git a/src/test/java/net/consensys/orion/network/CertificateAuthorityNodeClientTest.java b/src/test/java/net/consensys/orion/network/CertificateAuthorityNodeClientTest.java index d2201b09..b546708c 100644 --- a/src/test/java/net/consensys/orion/network/CertificateAuthorityNodeClientTest.java +++ b/src/test/java/net/consensys/orion/network/CertificateAuthorityNodeClientTest.java @@ -73,7 +73,7 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { final Router dummyRouter = Router.router(vertx); final ReadOnlyNetworkNodes payload = - new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyList(), Collections.emptyMap()); + new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyMap()); dummyRouter.post("/partyinfo").handler(routingContext -> { routingContext.response().end(Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, payload))); }); diff --git a/src/test/java/net/consensys/orion/network/InsecureNodeClientTest.java b/src/test/java/net/consensys/orion/network/InsecureNodeClientTest.java index d203a3cf..4b46fd6a 100644 --- a/src/test/java/net/consensys/orion/network/InsecureNodeClientTest.java +++ b/src/test/java/net/consensys/orion/network/InsecureNodeClientTest.java @@ -75,7 +75,7 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { final Router dummyRouter = Router.router(vertx); final ReadOnlyNetworkNodes payload = - new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyList(), Collections.emptyMap()); + new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyMap()); dummyRouter.post("/partyinfo").handler(routingContext -> { routingContext.response().end(Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, payload))); }); diff --git a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java index 3847ca2a..f3c0dd0f 100644 --- a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java +++ b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java @@ -108,9 +108,8 @@ void networkDiscoveryWithUnresponsivePeer() throws Exception { @Test void networkDiscoveryWithMerge() throws Exception { // empty memory nodes, lets' say one peer is alone in his network - final byte[] unknownPeerNetworkNodes = Serializer.serialize( - CBOR, - new ReadOnlyNetworkNodes(URI.create("http://localhost/"), Collections.emptyList(), Collections.emptyMap())); + final byte[] unknownPeerNetworkNodes = + Serializer.serialize(CBOR, new ReadOnlyNetworkNodes(URI.create("http://localhost/"), Collections.emptyMap())); final Buffer unknownPeerBody = new Buffer(); unknownPeerBody.write(unknownPeerNetworkNodes); // create a peer that's not in our current network nodes @@ -120,7 +119,6 @@ void networkDiscoveryWithMerge() throws Exception { // create a peer that we know, and that knows the lonely unknown peer. final ReadOnlyNetworkNodes knownPeerNetworkNodes = new ReadOnlyNetworkNodes( URI.create("http://www.example.com"), - Collections.singletonList(unknownPeer.getURI()), Collections.singletonMap(unknownPeer.publicKey, unknownPeer.getURI())); final Buffer knownPeerBody = new Buffer(); knownPeerBody.write(Serializer.serialize(CBOR, knownPeerNetworkNodes)); @@ -142,7 +140,9 @@ void networkDiscoveryWithMerge() throws Exception { final NetworkDiscovery.Discoverer knownPeerDiscoverer = networkDiscovery.discoverers().get(knownPeer.getURI()); assertNotNull(knownPeerDiscoverer); - Thread.sleep(3 * (knownPeerDiscoverer.currentRefreshDelay + 2000)); + System.out.println("Hitting known peer at " + knownPeer.getURI()); + System.out.println("Sleeping for " + (6 * (knownPeerDiscoverer.currentRefreshDelay + 2000))); + Thread.sleep(6 * (knownPeerDiscoverer.currentRefreshDelay + 2000)); // ensure knownPeer responded and that his party info was called at least twice assertTrue( @@ -155,6 +155,7 @@ void networkDiscoveryWithMerge() throws Exception { Iterator> iter = networkNodes.nodePKs().iterator(); while (iter.hasNext()) { size++; + iter.next(); } assertEquals(2, size); assertEquals(unknownPeer.getURI(), networkNodes.uriForRecipient(unknownPeer.publicKey)); diff --git a/src/test/java/net/consensys/orion/network/TofuNodeClientTest.java b/src/test/java/net/consensys/orion/network/TofuNodeClientTest.java index f34c6c41..88597b3f 100644 --- a/src/test/java/net/consensys/orion/network/TofuNodeClientTest.java +++ b/src/test/java/net/consensys/orion/network/TofuNodeClientTest.java @@ -82,7 +82,7 @@ void setUp() throws Exception { final Router dummyRouter = Router.router(vertx); final ReadOnlyNetworkNodes payload = - new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyList(), Collections.emptyMap()); + new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyMap()); dummyRouter.post("/partyinfo").handler(routingContext -> { routingContext.response().end(Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, payload))); }); diff --git a/src/test/java/net/consensys/orion/network/WhitelistNodeClientTest.java b/src/test/java/net/consensys/orion/network/WhitelistNodeClientTest.java index 5797b42f..7dea604f 100644 --- a/src/test/java/net/consensys/orion/network/WhitelistNodeClientTest.java +++ b/src/test/java/net/consensys/orion/network/WhitelistNodeClientTest.java @@ -83,7 +83,7 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { client = NodeHttpClientBuilder.build(vertx, config, 100); final ReadOnlyNetworkNodes payload = - new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyList(), Collections.emptyMap()); + new ReadOnlyNetworkNodes(URI.create("http://www.example.com"), Collections.emptyMap()); dummyRouter.post("/partyinfo").handler(routingContext -> { routingContext.response().end(Buffer.buffer(Serializer.serialize(HttpContentType.CBOR, payload))); }); From cbddc905229ae1496f7e6805a2c7e33390a26624 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Sun, 16 Feb 2020 11:19:46 -0800 Subject: [PATCH 06/19] Fix issues with json serialization --- .../net/consensys/orion/network/NetworkNodes.java | 2 +- .../orion/network/PersistentNetworkNodes.java | 12 ++---------- .../orion/network/ReadOnlyNetworkNodes.java | 2 +- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/consensys/orion/network/NetworkNodes.java b/src/main/java/net/consensys/orion/network/NetworkNodes.java index 839d1189..f31c9e31 100644 --- a/src/main/java/net/consensys/orion/network/NetworkNodes.java +++ b/src/main/java/net/consensys/orion/network/NetworkNodes.java @@ -34,7 +34,7 @@ public interface NetworkNodes { /** * @return List of URIs of other nodes on the network */ - @JsonProperty("nodeURIs") + @JsonProperty("nodeURLs") Collection nodeURIs(); /** diff --git a/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java b/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java index a729ec1c..b945d78f 100644 --- a/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java +++ b/src/main/java/net/consensys/orion/network/PersistentNetworkNodes.java @@ -103,16 +103,8 @@ public URI uri() { @Override public Collection nodeURIs() { Set allURIs = new HashSet<>(); - List completions = new ArrayList<>(); - nodePKs.keysAsync().thenAccept(keys -> { - for (Box.PublicKey key : keys) { - completions.add(nodePKs.getAsync(key).thenAccept(allURIs::add)); - } - }); - try { - AsyncCompletion.allOf(completions).join(); - } catch (InterruptedException e) { - logger.warn("Timeout waiting to collect all URLs", e); + for (Map.Entry entry : nodePKs()) { + allURIs.add(entry.getValue()); } return allURIs; } diff --git a/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java b/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java index 01637451..710d3b75 100644 --- a/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java +++ b/src/main/java/net/consensys/orion/network/ReadOnlyNetworkNodes.java @@ -36,7 +36,7 @@ public class ReadOnlyNetworkNodes implements NetworkNodes { @JsonCreator public ReadOnlyNetworkNodes( @JsonProperty("uri") final URI uri, - @JsonProperty("nodeURIs") List nodeURIs, + @JsonProperty("nodeURLs") List nodeURIs, @JsonProperty("nodePKs") @JsonDeserialize( keyUsing = PublicKeyMapKeyDeserializer.class) final Map nodePKs) { this.uri = uri; From 2d8417696112293fd59dbb3bae8246f58d04657d Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Sun, 16 Feb 2020 23:49:47 -0800 Subject: [PATCH 07/19] Use different storage --- .../consensys/orion/acceptance/NodeUtils.java | 4 ++- .../oracle/OracleSendReceiveTest.java | 3 +- .../postgresql/PostgresqlSendReceiveTest.java | 3 +- .../acceptance/send/SingleNodeSendTest.java | 3 +- .../receive/DualNodesSendReceiveTest.java | 18 ++++++++--- .../receive/SingleNodeSendReceiveTest.java | 3 +- .../DualNodesPrivacyGroupsTest.java | 8 +++-- ...NodesSendReceiveUsingPrivacyGroupTest.java | 8 +++-- .../privacyGroup/QueryGroupRestartTest.java | 3 +- .../SingleNodeDualSendReceiveTest.java | 3 +- .../java/net/consensys/orion/cmd/Orion.java | 32 +++++++++++++------ .../net/consensys/orion/config/Config.java | 27 +++++++++++++--- .../orion/network/NetworkDiscovery.java | 19 +++++++---- .../orion/config/TomlConfigTest.java | 3 +- .../orion/network/NetworkDiscoveryTest.java | 2 +- .../storage/PrivacyGroupStorageTest.java | 2 +- 16 files changed, 100 insertions(+), 41 deletions(-) diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java b/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java index bdb52751..3fefda2a 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java @@ -70,7 +70,8 @@ public static Config nodeConfig( final String tls, final String tlsServerTrust, final String tlsClientTrust, - final String storage) throws IOException { + final String storage, + final String nodeStorage) throws IOException { final Path workDir = tempDir.resolve("acceptance").toAbsolutePath(); Files.createDirectories(workDir); @@ -85,6 +86,7 @@ public static Config nodeConfig( + "clientport = " + clientPort + "\n" + "clientnetworkinterface = \"" + clientNetworkInterface + "\"\n" + "storage = \"" + storage + "\"\n" + + "nodestorage = \"" + nodeStorage + "\"\n" + "publickeys = [" + pubKeys + "]\n" + "privatekeys = [" + privKeys + "]\n" + "workdir= \"" + workDir.toString() + "\"\n"; diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/oracle/OracleSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/oracle/OracleSendReceiveTest.java index b9142545..0f85e40f 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/oracle/OracleSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/oracle/OracleSendReceiveTest.java @@ -91,7 +91,8 @@ void setUp(@TempDirectory final Path tempDir) throws Exception { "off", "tofu", "tofu", - "sql:" + jdbcUrl); + "sql:" + jdbcUrl, + "memory"); vertx = vertx(); orionLauncher = NodeUtils.startOrion(config); diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/postgresql/PostgresqlSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/postgresql/PostgresqlSendReceiveTest.java index b34bbc7a..fea3294d 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/postgresql/PostgresqlSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/postgresql/PostgresqlSendReceiveTest.java @@ -83,7 +83,8 @@ void setUp(@TempDirectory final Path tempDir) throws Exception { "off", "tofu", "tofu", - "sql:" + jdbcUrl); + "sql:" + jdbcUrl, + "memory"); vertx = vertx(); orionLauncher = NodeUtils.startOrion(config); diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java index cba0ac3a..7764fe3a 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java @@ -74,7 +74,8 @@ static void setUpSingleNode(@TempDirectory final Path tempDir) throws Exception "off", "tofu", "tofu", - "leveldb:database/node1"); + "leveldb:database/node1", + "memory"); } diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java index b5d610ba..6ddfe7d1 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java @@ -90,12 +90,18 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { pubKeyStrings = pubKeys.stream().map(pub -> getStringFromResource(pub)).collect(Collectors.toList()); final List priKeyFileList = priKeys.stream().map(pub -> getPath(tempDir, pub)).collect(Collectors.toList()); - final String jdbcUrl = "jdbc:h2:" + tempDir.resolve("node2").toString(); + final String jdbcUrl = "jdbc:h2:" + tempDir.resolve("DualNodesSendReceiveTest").toString(); try (final Connection conn = DriverManager.getConnection(jdbcUrl)) { final Statement st = conn.createStatement(); st.executeUpdate("create table if not exists store(key char(60), value binary, primary key(key))"); } + final String jdbcUrlNodeInfo = "jdbc:h2:" + tempDir.resolve("DualNodesSendReceiveTestNodeInfo").toString(); + try (final Connection conn = DriverManager.getConnection(jdbcUrlNodeInfo)) { + final Statement st = conn.createStatement(); + st.executeUpdate("create table if not exists store(key char(60), value binary, primary key(key))"); + } + firstNodeConfig = NodeUtils.nodeConfig( tempDir, @@ -109,7 +115,8 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { "off", "tofu", "tofu", - "leveldb:database/node1"); + "leveldb:database/node1", + "memory"); secondNodeConfig = NodeUtils.nodeConfig( tempDir, 0, @@ -122,7 +129,8 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { "off", "tofu", "tofu", - "sql:" + jdbcUrl); + "sql:" + jdbcUrl, + "sql:" + jdbcUrlNodeInfo); vertx = vertx(); firstOrionLauncher = NodeUtils.startOrion(firstNodeConfig); firstHttpClient = vertx.createHttpClient(); @@ -177,11 +185,11 @@ private String getStringFromResource(final String resourceFileName) { private ReadOnlyNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, final Request request) throws Exception { final Response resp = httpClient.newCall(request).execute(); - System.out.println(resp.body().string()); + assertEquals("Received " + resp.code(), 200, resp.code()); final ReadOnlyNetworkNodes partyInfoResponse = - Serializer.deserialize(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, resp.body().bytes()); + Serializer.deserialize(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, resp.body().bytes()); return partyInfoResponse; } diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/SingleNodeSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/SingleNodeSendReceiveTest.java index d49990e0..e8026090 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/SingleNodeSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/SingleNodeSendReceiveTest.java @@ -74,7 +74,8 @@ static void setUpSingleNode(@TempDirectory final Path tempDir) throws Exception "off", "tofu", "tofu", - "leveldb:database/node1"); + "leveldb:database/node1", + "memory"); } @BeforeEach diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java index 431f648f..8e8b07a2 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java @@ -88,7 +88,7 @@ void setUpDualNodes(@TempDir final Path tempDir) throws Exception { final Path key1key = copyResource("key1.key", tempDir.resolve("key1.key")); final Path key2pub = copyResource("key2.pub", tempDir.resolve("key2.pub")); final Path key2key = copyResource("key2.key", tempDir.resolve("key2.key")); - final String jdbcUrl = "jdbc:h2:" + tempDir.resolve("node2").toString(); + final String jdbcUrl = "jdbc:h2:" + tempDir.resolve("DualNodesPrivacyGroupsTest").toString(); try (final Connection conn = DriverManager.getConnection(jdbcUrl)) { final Statement st = conn.createStatement(); st.executeUpdate("create table if not exists store(key char(60), value binary, primary key(key))"); @@ -106,7 +106,8 @@ void setUpDualNodes(@TempDir final Path tempDir) throws Exception { "off", "tofu", "tofu", - "leveldb:" + tempDir + "database/node1"); + "leveldb:" + tempDir + "database/DualNodesPrivacyGroupsTest", + "memory"); final Config secondNodeConfig = NodeUtils.nodeConfig( tempDir, 0, @@ -119,7 +120,8 @@ void setUpDualNodes(@TempDir final Path tempDir) throws Exception { "off", "tofu", "tofu", - "sql:" + jdbcUrl); + "sql:" + jdbcUrl, + "memory"); vertx = vertx(); firstOrionLauncher = NodeUtils.startOrion(firstNodeConfig); diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java index e39f7aba..d87a9b86 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesSendReceiveUsingPrivacyGroupTest.java @@ -86,7 +86,7 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { final Path key2pub = copyResource("key2.pub", tempDir.resolve("key2.pub")); final Path key2key = copyResource("key2.key", tempDir.resolve("key2.key")); - final String jdbcUrl = "jdbc:h2:" + tempDir.resolve("node2").toString(); + final String jdbcUrl = "jdbc:h2:" + tempDir.resolve("DualNodesSendReceiveUsingPrivacyGroupTest").toString(); try (final Connection conn = DriverManager.getConnection(jdbcUrl)) { final Statement st = conn.createStatement(); st.executeUpdate("create table if not exists store(key char(60), value binary, primary key(key))"); @@ -104,7 +104,8 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { "off", "tofu", "tofu", - "leveldb:database/node1"); + "leveldb:database/DualNodesSendReceiveUsingPrivacyGroupTest", + "memory"); secondNodeConfig = NodeUtils.nodeConfig( tempDir, 0, @@ -117,7 +118,8 @@ void setUpDualNodes(@TempDirectory final Path tempDir) throws Exception { "off", "tofu", "tofu", - "sql:" + jdbcUrl); + "sql:" + jdbcUrl, + "memory"); vertx = vertx(); firstOrionLauncher = NodeUtils.startOrion(firstNodeConfig); firstHttpClient = vertx.createHttpClient(); diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/QueryGroupRestartTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/QueryGroupRestartTest.java index 268acea9..f80f9a06 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/QueryGroupRestartTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/QueryGroupRestartTest.java @@ -71,7 +71,8 @@ static void setUpSingleNode(@TempDirectory final Path tempDir) throws Exception "off", "tofu", "tofu", - "leveldb:database/node1"); + "leveldb:database/node1", + "leveldb:database/node1NodeInfo"); } @BeforeEach diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/SingleNodeDualSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/SingleNodeDualSendReceiveTest.java index ddee9b95..2c48b327 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/SingleNodeDualSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/SingleNodeDualSendReceiveTest.java @@ -75,7 +75,8 @@ static void setUpSingleNode(@TempDirectory final Path tempDir) throws Exception "off", "tofu", "tofu", - "leveldb:database/node1"); + "leveldb:database/node1", + "memory"); } @BeforeEach diff --git a/src/main/java/net/consensys/orion/cmd/Orion.java b/src/main/java/net/consensys/orion/cmd/Orion.java index e1d76e1e..5db702d2 100644 --- a/src/main/java/net/consensys/orion/cmd/Orion.java +++ b/src/main/java/net/consensys/orion/cmd/Orion.java @@ -70,6 +70,7 @@ import java.util.Properties; import java.util.Scanner; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; @@ -97,6 +98,7 @@ import org.apache.tuweni.kv.KeyValueStore; import org.apache.tuweni.kv.LevelDBKeyValueStore; import org.apache.tuweni.kv.MapDBKeyValueStore; +import org.apache.tuweni.kv.MapKeyValueStore; import org.apache.tuweni.kv.ProxyKeyValueStore; public class Orion { @@ -114,6 +116,7 @@ public class Orion { private final Vertx vertx; private KeyValueStore storage; + private KeyValueStore nodeStorage; private NetworkDiscovery discovery; private HttpServer nodeHTTPServer; private HttpServer clientHTTPServer; @@ -300,6 +303,13 @@ public void stop() { log.error("Error closing storage", e); } } + if (nodeStorage != null) { + try { + nodeStorage.close(); + } catch (final IOException e) { + log.error("Error closing node storage", e); + } + } } public void run(final PrintStream out, final PrintStream err, final String... args) { @@ -343,9 +353,11 @@ public void run(final PrintStream out, final PrintStream err, final Config confi log.info("using working directory {}", workDir); // create our storage engine - storage = createStorage(config.storage(), workDir); + storage = createStorage(config.storage(), workDir, "routerdb"); + nodeStorage = createStorage(config.nodeStorage(), workDir, "nodedb"); - final PersistentNetworkNodes networkNodes = new PersistentNetworkNodes(config, keyStore.nodeKeys(), wrap(storage)); + final PersistentNetworkNodes networkNodes = + new PersistentNetworkNodes(config, keyStore.nodeKeys(), wrap(nodeStorage)); final Enclave enclave = new SodiumEnclave(keyStore); @@ -483,7 +495,6 @@ public void run(final PrintStream out, final PrintStream err, final Config confi new URI("http", null, config.nodeNetworkInterface(), nodeHTTPServer.actualPort(), null, null, null); networkNodes.setNodeUrl(nodeURI, keyStore.nodeKeys()); } - final CompletableFuture networkDiscoveryFuture = new CompletableFuture<>(); // start network discovery of other peers discovery = new NetworkDiscovery(networkNodes, config); @@ -556,24 +567,23 @@ private static URI bytesToURI(Bytes v) { return null; } - private KeyValueStore createStorage(final String storage, final Path storagePath) { - String db = "routerdb"; + private KeyValueStore createStorage(final String storage, final Path storagePath, String dbName) { final String[] storageOptions = storage.split(":", 2); if (storageOptions.length > 1) { - db = storageOptions[1]; + dbName = storageOptions[1]; } final Function bytesIdentityFn = Function.identity(); if (storage.toLowerCase().startsWith("mapdb")) { return MapDBKeyValueStore - .open(storagePath.resolve(db), bytesIdentityFn, bytesIdentityFn, bytesIdentityFn, bytesIdentityFn); + .open(storagePath.resolve(dbName), bytesIdentityFn, bytesIdentityFn, bytesIdentityFn, bytesIdentityFn); } else if (storage.toLowerCase().startsWith("leveldb")) { try { - return LevelDBKeyValueStore.open(storagePath.resolve(db)); + return LevelDBKeyValueStore.open(storagePath.resolve(dbName)); } catch (final IOException e) { - throw new OrionStartException("Couldn't create LevelDB store: " + db, e); + throw new OrionStartException("Couldn't create LevelDB store: " + dbName, e); } } else if (storage.toLowerCase().startsWith("sql")) { - final JpaEntityManagerProvider jpaEntityManagerProvider = new JpaEntityManagerProvider(db); + final JpaEntityManagerProvider jpaEntityManagerProvider = new JpaEntityManagerProvider(dbName); return ProxyKeyValueStore.open( EntityManagerKeyValueStore.open(jpaEntityManagerProvider::createEntityManager, Store.class, Store::getKey), Base64::decode, @@ -585,6 +595,8 @@ private KeyValueStore createStorage(final String storage, final Pa store.setValue(value.toArrayUnsafe()); return store; }); + } else if (storage.toLowerCase().equals("memory")) { + return MapKeyValueStore.open(new ConcurrentHashMap<>()); } else { throw new OrionStartException("unsupported storage mechanism: " + storage); } diff --git a/src/main/java/net/consensys/orion/config/Config.java b/src/main/java/net/consensys/orion/config/Config.java index f2c50895..25165ffd 100644 --- a/src/main/java/net/consensys/orion/config/Config.java +++ b/src/main/java/net/consensys/orion/config/Config.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; @@ -176,11 +177,11 @@ public Path workDir() { * * @return A list of other node URLs to connect to on startup. */ - public List otherNodes() { + public List otherNodes() { return configuration.getListOfString("othernodes").stream().map(urlString -> { try { - return new URL(urlString); - } catch (MalformedURLException e) { + return URI.create(urlString); + } catch (IllegalArgumentException e) { throw new IllegalStateException("key 'othernodes' should have been validated, yet it's invalid", e); } }).collect(Collectors.toList()); @@ -259,6 +260,24 @@ public String storage() { return System.getenv().getOrDefault("ORION_STORAGE", configuration.getString("storage")); } + /** + * Storage engine used to save node information. Options: + * + *
    + *
  • leveldb:path - LevelDB
    • + *
  • mapdb:path - MapDB
    • + *
  • sql:jdbcurl - Relational database
    • + *
  • memory - Contents are cleared when Orion exits
    • + *
+ * + * Default: "memory" + * + * @return Storage string specifying a storage engine and/or storage path + */ + public String nodeStorage() { + return System.getenv().getOrDefault("ORION_NODE_STORAGE", configuration.getString("nodestorage")); + } + /** * TLS status. Options: * @@ -560,7 +579,7 @@ private static Schema configSchema() { Config::validateStorage); schemaBuilder.addString( - "nodeStorage", + "nodestorage", "memory", "Storage engine used to save node information. Options:\n" + "\n" diff --git a/src/main/java/net/consensys/orion/network/NetworkDiscovery.java b/src/main/java/net/consensys/orion/network/NetworkDiscovery.java index bfdb5f23..9dc9ab52 100644 --- a/src/main/java/net/consensys/orion/network/NetworkDiscovery.java +++ b/src/main/java/net/consensys/orion/network/NetworkDiscovery.java @@ -85,20 +85,27 @@ public void stop() { private void updateDiscoverers() { log.trace("Updating discoverers"); // for each peer that we know, we start a Discoverer (on timer) + for (final URI fixedNode : config.otherNodes()) { + discoverers.computeIfAbsent(fixedNode, this::createDiscoverer); + } + for (final URI nodeUri : nodes.nodeURIs()) { - discoverers.computeIfAbsent(nodeUri, uri -> { - log.trace("New discoverer for {}", uri); - final Discoverer d = new Discoverer(uri, refreshDelayMs, uri.equals(nodes.uri())); - d.engageNextTimerTick(); - return d; - }); + discoverers.computeIfAbsent(nodeUri, this::createDiscoverer); } + } public Map discoverers() { return new HashMap<>(discoverers); } + private Discoverer createDiscoverer(URI uri) { + log.trace("New discoverer for {}", uri); + final Discoverer d = new Discoverer(uri, refreshDelayMs, uri.equals(nodes.uri())); + d.engageNextTimerTick(); + return d; + } + /** * Discoverer handle() is fired by a timer * diff --git a/src/test/java/net/consensys/orion/config/TomlConfigTest.java b/src/test/java/net/consensys/orion/config/TomlConfigTest.java index 06132dfe..d87d1407 100644 --- a/src/test/java/net/consensys/orion/config/TomlConfigTest.java +++ b/src/test/java/net/consensys/orion/config/TomlConfigTest.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.InputStream; +import java.net.URI; import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; @@ -72,7 +73,7 @@ void fullFileRead() throws Exception { assertEquals(Collections.emptyList(), testConf.clientConnectionTlsServerChain()); // URL Array - assertEquals(Collections.singletonList(new URL("http://127.0.0.1:9000/")), testConf.otherNodes()); + assertEquals(Collections.singletonList(URI.create("http://127.0.0.1:9000/")), testConf.otherNodes()); } @Test diff --git a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java index f3c0dd0f..f6685421 100644 --- a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java +++ b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java @@ -49,7 +49,7 @@ class NetworkDiscoveryTest { private KeyValueStore store; @BeforeEach - void setUp() throws Exception { + void setUp() { vertx = Vertx.vertx(); config = Config.load("tls='off'"); store = MapKeyValueStore.open(new ConcurrentHashMap<>()); diff --git a/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java b/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java index 57df5b60..ca4dee39 100644 --- a/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java +++ b/src/test/java/net/consensys/orion/storage/PrivacyGroupStorageTest.java @@ -58,7 +58,7 @@ void setup(@TempDirectory final Path tempDir) throws SQLException { memoryKeyStore.addNodeKey(defaultNodeKey); enclave = new SodiumEnclave(memoryKeyStore); - final String jdbcUrl = "jdbc:h2:" + tempDir.resolve("node2").toString(); + final String jdbcUrl = "jdbc:h2:" + tempDir.resolve("PrivacyGroupStorageTest").toString(); try (final Connection conn = DriverManager.getConnection(jdbcUrl)) { final Statement st = conn.createStatement(); st.executeUpdate("create table if not exists store(key char(60), value binary, primary key(key))"); From 8205ef68b02d7a1673fe606b23afc0e4b99e0502 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Mon, 17 Feb 2020 00:40:19 -0800 Subject: [PATCH 08/19] prettify diff --- gradle/check-licenses.gradle | 2 +- .../acceptance/send/SingleNodeSendTest.java | 1 + .../java/net/consensys/orion/cmd/Orion.java | 21 +--------- .../consensys/orion/storage/StorageUtils.java | 39 +++++++++++++++++++ .../orion/http/handler/HandlerTest.java | 3 +- .../payload/DistributePayloadManagerTest.java | 3 +- 6 files changed, 47 insertions(+), 22 deletions(-) create mode 100644 src/main/java/net/consensys/orion/storage/StorageUtils.java diff --git a/gradle/check-licenses.gradle b/gradle/check-licenses.gradle index 1f1bc23a..e03df739 100644 --- a/gradle/check-licenses.gradle +++ b/gradle/check-licenses.gradle @@ -130,7 +130,7 @@ downloadLicenses { 'Eclipse Distribution License - v 1.0', ], (epl1): [ - 'Eclipse Public License - v 1.0' + 'Eclipse Public License - v 1.0', ], (epl2): [ 'Eclipse Public License v2.0', diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java index 7764fe3a..f041f4e1 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java @@ -25,6 +25,7 @@ import net.consensys.orion.exception.OrionErrorCode; import java.net.MalformedURLException; +import java.net.URI; import java.nio.file.Path; import io.vertx.core.Vertx; diff --git a/src/main/java/net/consensys/orion/cmd/Orion.java b/src/main/java/net/consensys/orion/cmd/Orion.java index 5db702d2..407c034d 100644 --- a/src/main/java/net/consensys/orion/cmd/Orion.java +++ b/src/main/java/net/consensys/orion/cmd/Orion.java @@ -53,6 +53,7 @@ import net.consensys.orion.storage.Sha512_256StorageKeyBuilder; import net.consensys.orion.storage.Storage; import net.consensys.orion.storage.StorageKeyBuilder; +import net.consensys.orion.storage.StorageUtils; import net.consensys.orion.storage.Store; import net.consensys.orion.utils.TLS; @@ -357,7 +358,7 @@ public void run(final PrintStream out, final PrintStream err, final Config confi nodeStorage = createStorage(config.nodeStorage(), workDir, "nodedb"); final PersistentNetworkNodes networkNodes = - new PersistentNetworkNodes(config, keyStore.nodeKeys(), wrap(nodeStorage)); + new PersistentNetworkNodes(config, keyStore.nodeKeys(), StorageUtils.convertToPubKeyStore(nodeStorage)); final Enclave enclave = new SodiumEnclave(keyStore); @@ -549,24 +550,6 @@ private Handler> completeFutureInHandler(final Completab }; } - public static KeyValueStore wrap(KeyValueStore store) { - return ProxyKeyValueStore - .open(store, Box.PublicKey::fromBytes, Box.PublicKey::bytes, Orion::bytesToURI, Orion::uriToBytes); - } - - private static Bytes uriToBytes(Box.PublicKey key, URI uri) { - return Bytes.wrap(uri.toString().getBytes(StandardCharsets.UTF_8)); - } - - private static URI bytesToURI(Bytes v) { - try { - return URI.create(new String(v.toArray(), StandardCharsets.UTF_8)); - } catch (IllegalArgumentException e) { - log.warn("Error reading URI", e); - } - return null; - } - private KeyValueStore createStorage(final String storage, final Path storagePath, String dbName) { final String[] storageOptions = storage.split(":", 2); if (storageOptions.length > 1) { diff --git a/src/main/java/net/consensys/orion/storage/StorageUtils.java b/src/main/java/net/consensys/orion/storage/StorageUtils.java new file mode 100644 index 00000000..908015a8 --- /dev/null +++ b/src/main/java/net/consensys/orion/storage/StorageUtils.java @@ -0,0 +1,39 @@ +package net.consensys.orion.storage; + +import net.consensys.orion.cmd.Orion; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.crypto.sodium.Box; +import org.apache.tuweni.kv.KeyValueStore; +import org.apache.tuweni.kv.ProxyKeyValueStore; + +import java.net.URI; +import java.nio.charset.StandardCharsets; + +/** Utility functions used to manipulate key-value store to expose higher order functions. **/ +public class StorageUtils { + + private static final Logger log = LogManager.getLogger(StorageUtils.class); + + private StorageUtils() { + } + + public static KeyValueStore convertToPubKeyStore(KeyValueStore store) { + return ProxyKeyValueStore + .open(store, Box.PublicKey::fromBytes, Box.PublicKey::bytes, StorageUtils::bytesToURI, StorageUtils::uriToBytes); + } + + private static Bytes uriToBytes(Box.PublicKey key, URI uri) { + return Bytes.wrap(uri.toString().getBytes(StandardCharsets.UTF_8)); + } + + private static URI bytesToURI(Bytes v) { + try { + return URI.create(new String(v.toArray(), StandardCharsets.UTF_8)); + } catch (IllegalArgumentException e) { + log.warn("Error reading URI", e); + } + return null; + } +} diff --git a/src/test/java/net/consensys/orion/http/handler/HandlerTest.java b/src/test/java/net/consensys/orion/http/handler/HandlerTest.java index 73b56c99..03a5941c 100644 --- a/src/test/java/net/consensys/orion/http/handler/HandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/HandlerTest.java @@ -31,6 +31,7 @@ import net.consensys.orion.storage.Sha512_256StorageKeyBuilder; import net.consensys.orion.storage.Storage; import net.consensys.orion.storage.StorageKeyBuilder; +import net.consensys.orion.storage.StorageUtils; import net.consensys.orion.utils.Serializer; import java.io.IOException; @@ -103,7 +104,7 @@ void setUp(@TempDirectory final Path tempDir) throws Exception { config = Config.load("tls='off'\nworkdir=\"" + tempDir + "\""); storage = MapKeyValueStore.open(); networkNodes = - new PersistentNetworkNodes(config, new Box.PublicKey[] {Box.KeyPair.random().publicKey()}, Orion.wrap(storage)); + new PersistentNetworkNodes(config, new Box.PublicKey[] {Box.KeyPair.random().publicKey()}, StorageUtils.convertToPubKeyStore(storage)); enclave = buildEnclave(tempDir); // create our vertx object diff --git a/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java b/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java index 042ee33d..f08f73f5 100644 --- a/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java +++ b/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java @@ -46,6 +46,7 @@ import net.consensys.orion.storage.Sha512_256StorageKeyBuilder; import net.consensys.orion.storage.Storage; import net.consensys.orion.storage.StorageKeyBuilder; +import net.consensys.orion.storage.StorageUtils; import net.consensys.orion.utils.Serializer; import java.nio.file.Path; @@ -104,7 +105,7 @@ public void beforeEach(@TempDirectory final Path tempDir) throws Exception { payloadStorage = new EncryptedPayloadStorage(storage, keyBuilder); privacyGroupStorage = new PrivacyGroupStorage(storage, enclave); queryPrivacyGroupStorage = new QueryPrivacyGroupStorage(storage, enclave); - networkNodes = new PersistentNetworkNodes(config, enclave.nodeKeys(), Orion.wrap(storage)); + networkNodes = new PersistentNetworkNodes(config, enclave.nodeKeys(), StorageUtils.convertToPubKeyStore(storage)); distributePayloadManager = new DistributePayloadManager( enclave, From c3bb3396ebb3f0fe8f3f5c6a83e75cd0e6eec22c Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Mon, 17 Feb 2020 00:53:22 -0800 Subject: [PATCH 09/19] spotless --- .../acceptance/send/SingleNodeSendTest.java | 1 - .../receive/DualNodesSendReceiveTest.java | 2 +- .../java/net/consensys/orion/cmd/Orion.java | 2 - .../orion/network/NetworkDiscovery.java | 1 - .../consensys/orion/storage/StorageUtils.java | 56 ++++++++++++------- .../orion/http/handler/HandlerTest.java | 6 +- .../orion/network/NetworkDiscoveryTest.java | 4 +- .../payload/DistributePayloadManagerTest.java | 1 - 8 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java index f041f4e1..7764fe3a 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java @@ -25,7 +25,6 @@ import net.consensys.orion.exception.OrionErrorCode; import java.net.MalformedURLException; -import java.net.URI; import java.nio.file.Path; import io.vertx.core.Vertx; diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java index 6ddfe7d1..c781958f 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java @@ -189,7 +189,7 @@ private ReadOnlyNetworkNodes getPartyInfoResponse(final OkHttpClient httpClient, assertEquals("Received " + resp.code(), 200, resp.code()); final ReadOnlyNetworkNodes partyInfoResponse = - Serializer.deserialize(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, resp.body().bytes()); + Serializer.deserialize(HttpContentType.CBOR, ReadOnlyNetworkNodes.class, resp.body().bytes()); return partyInfoResponse; } diff --git a/src/main/java/net/consensys/orion/cmd/Orion.java b/src/main/java/net/consensys/orion/cmd/Orion.java index 407c034d..5640c3d4 100644 --- a/src/main/java/net/consensys/orion/cmd/Orion.java +++ b/src/main/java/net/consensys/orion/cmd/Orion.java @@ -63,7 +63,6 @@ import java.io.PrintStream; import java.net.URI; import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -92,7 +91,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes; -import org.apache.tuweni.crypto.sodium.Box; import org.apache.tuweni.crypto.sodium.Sodium; import org.apache.tuweni.io.Base64; import org.apache.tuweni.kv.EntityManagerKeyValueStore; diff --git a/src/main/java/net/consensys/orion/network/NetworkDiscovery.java b/src/main/java/net/consensys/orion/network/NetworkDiscovery.java index 9dc9ab52..6c4cf2a5 100644 --- a/src/main/java/net/consensys/orion/network/NetworkDiscovery.java +++ b/src/main/java/net/consensys/orion/network/NetworkDiscovery.java @@ -160,7 +160,6 @@ public void handle(final Long timerId) { }) .exceptionHandler(ex -> { log.error("calling partyInfo on {} failed {}", nodeUrl, ex.getMessage()); - log.error(ex.getMessage(), ex); engageNextTimerTick(); }) .putHeader("Content-Type", "application/cbor") diff --git a/src/main/java/net/consensys/orion/storage/StorageUtils.java b/src/main/java/net/consensys/orion/storage/StorageUtils.java index 908015a8..6b0b2ec7 100644 --- a/src/main/java/net/consensys/orion/storage/StorageUtils.java +++ b/src/main/java/net/consensys/orion/storage/StorageUtils.java @@ -1,6 +1,20 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ package net.consensys.orion.storage; -import net.consensys.orion.cmd.Orion; +import java.net.URI; +import java.nio.charset.StandardCharsets; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes; @@ -8,32 +22,32 @@ import org.apache.tuweni.kv.KeyValueStore; import org.apache.tuweni.kv.ProxyKeyValueStore; -import java.net.URI; -import java.nio.charset.StandardCharsets; - /** Utility functions used to manipulate key-value store to expose higher order functions. **/ public class StorageUtils { - private static final Logger log = LogManager.getLogger(StorageUtils.class); + private static final Logger log = LogManager.getLogger(StorageUtils.class); - private StorageUtils() { - } + private StorageUtils() {} - public static KeyValueStore convertToPubKeyStore(KeyValueStore store) { - return ProxyKeyValueStore - .open(store, Box.PublicKey::fromBytes, Box.PublicKey::bytes, StorageUtils::bytesToURI, StorageUtils::uriToBytes); - } + public static KeyValueStore convertToPubKeyStore(KeyValueStore store) { + return ProxyKeyValueStore.open( + store, + Box.PublicKey::fromBytes, + Box.PublicKey::bytes, + StorageUtils::bytesToURI, + StorageUtils::uriToBytes); + } - private static Bytes uriToBytes(Box.PublicKey key, URI uri) { - return Bytes.wrap(uri.toString().getBytes(StandardCharsets.UTF_8)); - } + private static Bytes uriToBytes(Box.PublicKey key, URI uri) { + return Bytes.wrap(uri.toString().getBytes(StandardCharsets.UTF_8)); + } - private static URI bytesToURI(Bytes v) { - try { - return URI.create(new String(v.toArray(), StandardCharsets.UTF_8)); - } catch (IllegalArgumentException e) { - log.warn("Error reading URI", e); - } - return null; + private static URI bytesToURI(Bytes v) { + try { + return URI.create(new String(v.toArray(), StandardCharsets.UTF_8)); + } catch (IllegalArgumentException e) { + log.warn("Error reading URI", e); } + return null; + } } diff --git a/src/test/java/net/consensys/orion/http/handler/HandlerTest.java b/src/test/java/net/consensys/orion/http/handler/HandlerTest.java index 03a5941c..ccfd1cb9 100644 --- a/src/test/java/net/consensys/orion/http/handler/HandlerTest.java +++ b/src/test/java/net/consensys/orion/http/handler/HandlerTest.java @@ -103,8 +103,10 @@ void setUp(@TempDirectory final Path tempDir) throws Exception { // orion dependencies, reset them all between tests config = Config.load("tls='off'\nworkdir=\"" + tempDir + "\""); storage = MapKeyValueStore.open(); - networkNodes = - new PersistentNetworkNodes(config, new Box.PublicKey[] {Box.KeyPair.random().publicKey()}, StorageUtils.convertToPubKeyStore(storage)); + networkNodes = new PersistentNetworkNodes( + config, + new Box.PublicKey[] {Box.KeyPair.random().publicKey()}, + StorageUtils.convertToPubKeyStore(storage)); enclave = buildEnclave(tempDir); // create our vertx object diff --git a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java index f6685421..66ec159c 100644 --- a/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java +++ b/src/test/java/net/consensys/orion/network/NetworkDiscoveryTest.java @@ -140,9 +140,7 @@ void networkDiscoveryWithMerge() throws Exception { final NetworkDiscovery.Discoverer knownPeerDiscoverer = networkDiscovery.discoverers().get(knownPeer.getURI()); assertNotNull(knownPeerDiscoverer); - System.out.println("Hitting known peer at " + knownPeer.getURI()); - System.out.println("Sleeping for " + (6 * (knownPeerDiscoverer.currentRefreshDelay + 2000))); - Thread.sleep(6 * (knownPeerDiscoverer.currentRefreshDelay + 2000)); + Thread.sleep(3 * (knownPeerDiscoverer.currentRefreshDelay + 2000)); // ensure knownPeer responded and that his party info was called at least twice assertTrue( diff --git a/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java b/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java index f08f73f5..661b0d15 100644 --- a/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java +++ b/src/test/java/net/consensys/orion/payload/DistributePayloadManagerTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import net.consensys.orion.cmd.Orion; import net.consensys.orion.config.Config; import net.consensys.orion.enclave.Enclave; import net.consensys.orion.enclave.EncryptedPayload; From 7decf1c6e4c2b3e4be69dce6f3c383c0850c8c10 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Mon, 17 Feb 2020 00:57:10 -0800 Subject: [PATCH 10/19] Fix EPL license aliases --- gradle/check-licenses.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gradle/check-licenses.gradle b/gradle/check-licenses.gradle index e03df739..bbd51145 100644 --- a/gradle/check-licenses.gradle +++ b/gradle/check-licenses.gradle @@ -128,9 +128,11 @@ downloadLicenses { ], (edl1): [ 'Eclipse Distribution License - v 1.0', + 'Eclipse Distribution License v. 1.0', ], (epl1): [ 'Eclipse Public License - v 1.0', + 'Eclipse Public License v1.0', ], (epl2): [ 'Eclipse Public License v2.0', From f2bf0e92d9229323558f3585ae0b36dc546bc7d2 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Mon, 17 Feb 2020 01:12:08 -0800 Subject: [PATCH 11/19] Longer timeout --- .../send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java index 8e8b07a2..0f2c538c 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/privacyGroup/DualNodesPrivacyGroupsTest.java @@ -147,7 +147,7 @@ void setUpDualNodes(@TempDir final Path tempDir) throws Exception { final String firstNodeBaseUrl = NodeUtils.urlString("127.0.0.1", firstOrionLauncher.nodePort()); final Request request = new Request.Builder().post(partyInfoBody).url(firstNodeBaseUrl + "/partyinfo").build(); // first /partyinfo call may just get the one node, so wait until we get at least 2 nodes - await().atMost(5, TimeUnit.SECONDS).until(() -> getPartyInfoResponse(httpClient, request).nodeURIs().size() == 2); + await().atMost(10, TimeUnit.SECONDS).until(() -> getPartyInfoResponse(httpClient, request).nodeURIs().size() == 2); } From a2dc6b0f2e7e67506a7255eae240844f6e3e9549 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Mon, 17 Feb 2020 21:27:45 -0800 Subject: [PATCH 12/19] add test for two persistent nodes reusing the same store --- .../network/PersistentNetworkNodesTest.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 src/test/java/net/consensys/orion/network/PersistentNetworkNodesTest.java diff --git a/src/test/java/net/consensys/orion/network/PersistentNetworkNodesTest.java b/src/test/java/net/consensys/orion/network/PersistentNetworkNodesTest.java new file mode 100644 index 00000000..64eae0bf --- /dev/null +++ b/src/test/java/net/consensys/orion/network/PersistentNetworkNodesTest.java @@ -0,0 +1,67 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ +package net.consensys.orion.network; + +import static org.apache.tuweni.crypto.sodium.Box.KeyPair.random; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import net.consensys.orion.config.Config; + +import java.net.URI; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.tuweni.crypto.sodium.Box; +import org.apache.tuweni.kv.MapKeyValueStore; +import org.junit.jupiter.api.Test; + +class PersistentNetworkNodesTest { + + @Test + void testTwoInstancesSameStore() { + Config config = Config.load("tls='off'"); + MapKeyValueStore store = MapKeyValueStore.open(new ConcurrentHashMap<>()); + PersistentNetworkNodes nodes1 = new PersistentNetworkNodes(config, new Box.PublicKey[0], store); + PersistentNetworkNodes nodes2 = new PersistentNetworkNodes(config, new Box.PublicKey[0], store); + + Box.PublicKey pk = random().publicKey(); + boolean changed = nodes1.addNode(Collections.singletonMap(pk, URI.create("http://example:com:56666")).entrySet()); + assertTrue(changed); + + Iterator> iter = nodes2.nodePKs().iterator(); + Map.Entry entry = iter.next(); + assertEquals(pk, entry.getKey()); + assertFalse(iter.hasNext()); + } + + @Test + void testTwoInstancesInSequence() { + Config config = Config.load("tls='off'"); + MapKeyValueStore store = MapKeyValueStore.open(new ConcurrentHashMap<>()); + PersistentNetworkNodes nodes1 = new PersistentNetworkNodes(config, new Box.PublicKey[0], store); + + Box.PublicKey pk = random().publicKey(); + boolean changed = nodes1.addNode(Collections.singletonMap(pk, URI.create("http://example:com:56666")).entrySet()); + assertTrue(changed); + + PersistentNetworkNodes nodes2 = new PersistentNetworkNodes(config, new Box.PublicKey[0], store); + Iterator> iter = nodes2.nodePKs().iterator(); + Map.Entry entry = iter.next(); + assertEquals(pk, entry.getKey()); + assertFalse(iter.hasNext()); + } +} From 648c2fc90787232eef84451b32523c6681e2a8cb Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Tue, 18 Feb 2020 16:20:00 -0800 Subject: [PATCH 13/19] code review changes --- .../java/net/consensys/orion/cmd/Orion.java | 22 ++++++++++++++----- .../net/consensys/orion/config/Config.java | 10 ++++----- .../network/PersistentNetworkNodesTest.java | 16 ++++++++++++++ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/consensys/orion/cmd/Orion.java b/src/main/java/net/consensys/orion/cmd/Orion.java index 5640c3d4..21399dce 100644 --- a/src/main/java/net/consensys/orion/cmd/Orion.java +++ b/src/main/java/net/consensys/orion/cmd/Orion.java @@ -67,6 +67,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.security.Security; +import java.util.ArrayList; +import java.util.List; import java.util.Properties; import java.util.Scanner; import java.util.concurrent.CompletableFuture; @@ -75,6 +77,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import javax.annotation.Nullable; +import javax.persistence.EntityManagerFactory; import io.vertx.core.AsyncResult; import io.vertx.core.Future; @@ -115,10 +118,11 @@ public class Orion { private final Vertx vertx; private KeyValueStore storage; - private KeyValueStore nodeStorage; + private KeyValueStore knownNodesStorage; private NetworkDiscovery discovery; private HttpServer nodeHTTPServer; private HttpServer clientHTTPServer; + private List entityManagerFactories = new ArrayList<>(); public static void main(final String[] args) { log.info("starting orion"); @@ -302,13 +306,18 @@ public void stop() { log.error("Error closing storage", e); } } - if (nodeStorage != null) { + + if (knownNodesStorage != null) { try { - nodeStorage.close(); + knownNodesStorage.close(); } catch (final IOException e) { log.error("Error closing node storage", e); } } + + for (EntityManagerFactory factory : entityManagerFactories) { + factory.close(); + } } public void run(final PrintStream out, final PrintStream err, final String... args) { @@ -353,10 +362,10 @@ public void run(final PrintStream out, final PrintStream err, final Config confi // create our storage engine storage = createStorage(config.storage(), workDir, "routerdb"); - nodeStorage = createStorage(config.nodeStorage(), workDir, "nodedb"); + knownNodesStorage = createStorage(config.knownNodesStorage(), workDir, "nodedb"); final PersistentNetworkNodes networkNodes = - new PersistentNetworkNodes(config, keyStore.nodeKeys(), StorageUtils.convertToPubKeyStore(nodeStorage)); + new PersistentNetworkNodes(config, keyStore.nodeKeys(), StorageUtils.convertToPubKeyStore(knownNodesStorage)); final Enclave enclave = new SodiumEnclave(keyStore); @@ -532,7 +541,7 @@ private void writePortsToFile(final Config config, final int nodePort, final int try (final FileOutputStream fileOutputStream = new FileOutputStream(portsFile)) { properties.store( fileOutputStream, - "This file contains the ports used by the running instance of Besu. This file will be deleted after the node is shutdown."); + "This file contains the ports used by the running instance of Orion. This file will be deleted after the node is shutdown."); } catch (final Exception e) { log.warn("Error writing ports file", e); } @@ -565,6 +574,7 @@ private KeyValueStore createStorage(final String storage, final Pa } } else if (storage.toLowerCase().startsWith("sql")) { final JpaEntityManagerProvider jpaEntityManagerProvider = new JpaEntityManagerProvider(dbName); + entityManagerFactories.add(jpaEntityManagerProvider); return ProxyKeyValueStore.open( EntityManagerKeyValueStore.open(jpaEntityManagerProvider::createEntityManager, Store.class, Store::getKey), Base64::decode, diff --git a/src/main/java/net/consensys/orion/config/Config.java b/src/main/java/net/consensys/orion/config/Config.java index 25165ffd..5643e259 100644 --- a/src/main/java/net/consensys/orion/config/Config.java +++ b/src/main/java/net/consensys/orion/config/Config.java @@ -261,7 +261,7 @@ public String storage() { } /** - * Storage engine used to save node information. Options: + * Storage engine used to save known nodes information. Options: * *
    *
  • leveldb:path - LevelDB
    • @@ -274,8 +274,8 @@ public String storage() { * * @return Storage string specifying a storage engine and/or storage path */ - public String nodeStorage() { - return System.getenv().getOrDefault("ORION_NODE_STORAGE", configuration.getString("nodestorage")); + public String knownNodesStorage() { + return System.getenv().getOrDefault("ORION_KNOWN_NODE_STORAGE", configuration.getString("knownnodesstorage")); } /** @@ -579,9 +579,9 @@ private static Schema configSchema() { Config::validateStorage); schemaBuilder.addString( - "nodestorage", + "knownnodesstorage", "memory", - "Storage engine used to save node information. Options:\n" + "Storage engine used to save known node information. Options:\n" + "\n" + " - leveldb:path - LevelDB\n" + " - mapdb:path - MapDB\n" diff --git a/src/test/java/net/consensys/orion/network/PersistentNetworkNodesTest.java b/src/test/java/net/consensys/orion/network/PersistentNetworkNodesTest.java index 64eae0bf..bca5bd1f 100644 --- a/src/test/java/net/consensys/orion/network/PersistentNetworkNodesTest.java +++ b/src/test/java/net/consensys/orion/network/PersistentNetworkNodesTest.java @@ -64,4 +64,20 @@ void testTwoInstancesInSequence() { assertEquals(pk, entry.getKey()); assertFalse(iter.hasNext()); } + + @Test + void tryOverridingNodeInfo() { + Config config = Config.load("tls='off'"); + MapKeyValueStore store = MapKeyValueStore.open(new ConcurrentHashMap<>()); + PersistentNetworkNodes nodes = new PersistentNetworkNodes(config, new Box.PublicKey[0], store); + Box.PublicKey pk = random().publicKey(); + boolean changed = nodes.addNode(Collections.singletonMap(pk, URI.create("http://example:com:56666")).entrySet()); + assertTrue(changed); + + changed = nodes.addNode(Collections.singletonMap(pk, URI.create("http://evil:com:56666")).entrySet()); + assertFalse(changed); + + assertEquals(URI.create("http://example:com:56666"), nodes.uriForRecipient(pk)); + + } } From 85d389f5ee9a7ef573cf52612ffd6ae5a1c4f1dc Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Tue, 18 Feb 2020 16:24:24 -0800 Subject: [PATCH 14/19] fixing wrong cast --- src/main/java/net/consensys/orion/cmd/Orion.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/consensys/orion/cmd/Orion.java b/src/main/java/net/consensys/orion/cmd/Orion.java index 21399dce..033da733 100644 --- a/src/main/java/net/consensys/orion/cmd/Orion.java +++ b/src/main/java/net/consensys/orion/cmd/Orion.java @@ -77,7 +77,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import javax.annotation.Nullable; -import javax.persistence.EntityManagerFactory; import io.vertx.core.AsyncResult; import io.vertx.core.Future; @@ -122,7 +121,7 @@ public class Orion { private NetworkDiscovery discovery; private HttpServer nodeHTTPServer; private HttpServer clientHTTPServer; - private List entityManagerFactories = new ArrayList<>(); + private List entityManagerFactories = new ArrayList<>(); public static void main(final String[] args) { log.info("starting orion"); @@ -315,8 +314,8 @@ public void stop() { } } - for (EntityManagerFactory factory : entityManagerFactories) { - factory.close(); + for (JpaEntityManagerProvider provider : entityManagerFactories) { + provider.close(); } } From e5c11993185f22398b5b84c1ee1a02746fbdf717 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Thu, 20 Feb 2020 18:07:15 -0800 Subject: [PATCH 15/19] Allow to clear known nodes with a CLI flag --- .../java/net/consensys/orion/cmd/Orion.java | 9 ++++--- .../consensys/orion/cmd/OrionArguments.java | 11 +++++++- .../orion/cmd/OrionArgumentsTest.java | 26 ++++++++++++++++++- .../net/consensys/orion/cmd/OrionTest.java | 2 +- .../orion/http/CAOrTofuSecurityTest.java | 2 +- .../CertificateAuthoritySecurityTest.java | 2 +- .../orion/http/InsecureSecurityTest.java | 2 +- .../orion/http/TofuSecurityTest.java | 2 +- .../orion/http/WhiteListSecurityTest.java | 2 +- 9 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/consensys/orion/cmd/Orion.java b/src/main/java/net/consensys/orion/cmd/Orion.java index 033da733..0c261df1 100644 --- a/src/main/java/net/consensys/orion/cmd/Orion.java +++ b/src/main/java/net/consensys/orion/cmd/Orion.java @@ -116,12 +116,12 @@ public class Orion { } private final Vertx vertx; + private final List entityManagerFactories = new ArrayList<>(); private KeyValueStore storage; private KeyValueStore knownNodesStorage; private NetworkDiscovery discovery; private HttpServer nodeHTTPServer; private HttpServer clientHTTPServer; - private List entityManagerFactories = new ArrayList<>(); public static void main(final String[] args) { log.info("starting orion"); @@ -340,10 +340,10 @@ public void run(final PrintStream out, final PrintStream err, final String... ar return; } - run(out, err, config); + run(config, arguments.clearKnownNodes()); } - public void run(final PrintStream out, final PrintStream err, final Config config) { + public void run(final Config config, final boolean clearKnownNodes) { final Path libSodiumPath = config.libSodiumPath(); if (libSodiumPath != null) { Sodium.loadLibrary(libSodiumPath); @@ -362,6 +362,9 @@ public void run(final PrintStream out, final PrintStream err, final Config confi // create our storage engine storage = createStorage(config.storage(), workDir, "routerdb"); knownNodesStorage = createStorage(config.knownNodesStorage(), workDir, "nodedb"); + if (clearKnownNodes) { + knownNodesStorage.clearAsync(); + } final PersistentNetworkNodes networkNodes = new PersistentNetworkNodes(config, keyStore.nodeKeys(), StorageUtils.convertToPubKeyStore(knownNodesStorage)); diff --git a/src/main/java/net/consensys/orion/cmd/OrionArguments.java b/src/main/java/net/consensys/orion/cmd/OrionArguments.java index aef117fa..01cba59d 100644 --- a/src/main/java/net/consensys/orion/cmd/OrionArguments.java +++ b/src/main/java/net/consensys/orion/cmd/OrionArguments.java @@ -24,6 +24,7 @@ class OrionArguments { private boolean argumentExit = false; + private boolean clearKnownNodes = false; private Optional configFileName = Optional.empty(); private Optional keysToGenerate = Optional.empty(); @@ -58,6 +59,9 @@ class OrionArguments { displayVersion(out, err); argumentExit = true; break; + case "--clear-known-nodes": + clearKnownNodes = true; + break; default: if (args[i].startsWith("-")) { err.printf("Invalid option: %s\n", args[i]); @@ -79,8 +83,9 @@ private void displayHelp(final PrintStream out) { out.println("where options include:"); out.println("\t-g"); out.println("\t--generatekeys "); - out.println("\t\tgenerate key pairs for each of the names supplied."); + out.println("\t\tgenerate key pairs for each of the names supplied"); out.println("\t\twhere are a comma-separated list"); + out.println("\t--clear-known-nodes\tclear known nodes information."); out.println("\t-h"); out.println("\t--help\tprint this help message"); out.println("\t-v"); @@ -105,6 +110,10 @@ boolean argumentExit() { return argumentExit; } + boolean clearKnownNodes() { + return clearKnownNodes; + } + Optional configFileName() { return configFileName; } diff --git a/src/test/java/net/consensys/orion/cmd/OrionArgumentsTest.java b/src/test/java/net/consensys/orion/cmd/OrionArgumentsTest.java index 53b1bb2c..7e80e91f 100644 --- a/src/test/java/net/consensys/orion/cmd/OrionArgumentsTest.java +++ b/src/test/java/net/consensys/orion/cmd/OrionArgumentsTest.java @@ -14,6 +14,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.BufferedReader; @@ -33,7 +34,8 @@ class OrionArgumentsTest { + " [options] [config file]%n" + "where options include:%n" + "\t-g%n" - + "\t--generatekeys %n\t\tgenerate key pairs for each of the names supplied.%n\t\twhere are a comma-separated list%n" + + "\t--generatekeys %n\t\tgenerate key pairs for each of the names supplied%n\t\twhere are a comma-separated list%n" + + "\t--clear-known-nodes\tclear known nodes information.%n" + "\t-h%n" + "\t--help\tprint this help message%n" + "\t-v%n" @@ -92,6 +94,28 @@ void invalidOption() { assertTrue(arguments.argumentExit()); } + @Test + void clearKnownNodeDefaultOption() { + final String[] args = {}; + + final OrionArguments arguments = new OrionArguments(outStream, errStream, args); + + assertEquals("", errContent.toString()); + assertEquals("", outContent.toString()); + assertFalse(arguments.clearKnownNodes()); + } + + @Test + void clearKnownNodes() { + final String[] args = {"--clear-known-nodes"}; + + final OrionArguments arguments = new OrionArguments(outStream, errStream, args); + + assertEquals("", errContent.toString()); + assertEquals("", outContent.toString()); + assertTrue(arguments.clearKnownNodes()); + } + @Test void validAndInvalidOptions() { final String errorMsg = "Invalid option: -x\n"; diff --git a/src/test/java/net/consensys/orion/cmd/OrionTest.java b/src/test/java/net/consensys/orion/cmd/OrionTest.java index b9c9d9b6..7149b812 100644 --- a/src/test/java/net/consensys/orion/cmd/OrionTest.java +++ b/src/test/java/net/consensys/orion/cmd/OrionTest.java @@ -150,6 +150,6 @@ void startupFails(@TempDirectory final Path tempDir) { final Orion orion = new Orion(vertx); final Config config = Config.load("workdir=\"" + tempDir.resolve("data") + "\"\ntls=\"off\"\n"); - assertThrows(OrionStartException.class, () -> orion.run(System.out, System.err, config)); + assertThrows(OrionStartException.class, () -> orion.run(config, false)); } } diff --git a/src/test/java/net/consensys/orion/http/CAOrTofuSecurityTest.java b/src/test/java/net/consensys/orion/http/CAOrTofuSecurityTest.java index 7a445b20..6fac8a52 100644 --- a/src/test/java/net/consensys/orion/http/CAOrTofuSecurityTest.java +++ b/src/test/java/net/consensys/orion/http/CAOrTofuSecurityTest.java @@ -81,7 +81,7 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { new HttpClientOptions().setSsl(true).setTrustAll(true).setKeyCertOptions(clientCert.keyCertOptions())); orion = new Orion(vertx); - orion.run(System.out, System.err, config); + orion.run(config, false); } @AfterAll diff --git a/src/test/java/net/consensys/orion/http/CertificateAuthoritySecurityTest.java b/src/test/java/net/consensys/orion/http/CertificateAuthoritySecurityTest.java index fdb762be..aed7ba33 100644 --- a/src/test/java/net/consensys/orion/http/CertificateAuthoritySecurityTest.java +++ b/src/test/java/net/consensys/orion/http/CertificateAuthoritySecurityTest.java @@ -69,7 +69,7 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { new HttpClientOptions().setSsl(true).setTrustAll(true).setKeyCertOptions(clientCert.keyCertOptions())); orion = new Orion(vertx); - orion.run(System.out, System.err, config); + orion.run(config, false); } @AfterAll diff --git a/src/test/java/net/consensys/orion/http/InsecureSecurityTest.java b/src/test/java/net/consensys/orion/http/InsecureSecurityTest.java index aae6ca79..53d7c7ab 100644 --- a/src/test/java/net/consensys/orion/http/InsecureSecurityTest.java +++ b/src/test/java/net/consensys/orion/http/InsecureSecurityTest.java @@ -76,7 +76,7 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { .createHttpClient(new HttpClientOptions().setSsl(true).setKeyCertOptions(clientCertificate.keyCertOptions())); orion = new Orion(vertx); - orion.run(System.out, System.err, config); + orion.run(config, false); } @AfterAll diff --git a/src/test/java/net/consensys/orion/http/TofuSecurityTest.java b/src/test/java/net/consensys/orion/http/TofuSecurityTest.java index 214a4fb9..9d4f5d90 100644 --- a/src/test/java/net/consensys/orion/http/TofuSecurityTest.java +++ b/src/test/java/net/consensys/orion/http/TofuSecurityTest.java @@ -84,7 +84,7 @@ void setUp(@TempDirectory final Path tempDir) throws Exception { new HttpClientOptions().setSsl(true).setKeyCertOptions(anotherExampleDotComCert.keyCertOptions())); orion = new Orion(vertx); - orion.run(System.out, System.err, config); + orion.run(config, false); } @AfterEach diff --git a/src/test/java/net/consensys/orion/http/WhiteListSecurityTest.java b/src/test/java/net/consensys/orion/http/WhiteListSecurityTest.java index 659b0ec6..4516e754 100644 --- a/src/test/java/net/consensys/orion/http/WhiteListSecurityTest.java +++ b/src/test/java/net/consensys/orion/http/WhiteListSecurityTest.java @@ -93,7 +93,7 @@ static void setUp(@TempDirectory final Path tempDir) throws Exception { new HttpClientOptions().setSsl(true).setKeyCertOptions(anotherExampleDotComCert.keyCertOptions())); orion = new Orion(vertx); - orion.run(System.out, System.err, config); + orion.run(config, false); } @AfterAll From 9b87d206fb875c9905432971db72d17d3a1c44fd Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Thu, 20 Feb 2020 18:16:56 -0800 Subject: [PATCH 16/19] fix acceptance test --- .../java/net/consensys/orion/acceptance/NodeUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java b/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java index 3fefda2a..828e0f78 100644 --- a/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java +++ b/src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java @@ -104,7 +104,7 @@ public static int freePort() throws Exception { /** It's the callers responsibility to stop the started Orion. */ public static Orion startOrion(final Config config) { final Orion orion = new Orion(); - orion.run(System.out, System.err, config); + orion.run(config, false); return orion; } From 29cc6016cc13555e20b3aedce100d424cdf01a63 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Thu, 20 Feb 2020 23:10:30 -0800 Subject: [PATCH 17/19] Use env vars for all config keys --- .../net/consensys/orion/config/Config.java | 111 ++++++++++++++---- .../orion/config/TomlConfigTest.java | 33 +++++- src/test/resources/fullConfigTest.toml | 1 + 3 files changed, 117 insertions(+), 28 deletions(-) diff --git a/src/main/java/net/consensys/orion/config/Config.java b/src/main/java/net/consensys/orion/config/Config.java index 5643e259..bb1ad51c 100644 --- a/src/main/java/net/consensys/orion/config/Config.java +++ b/src/main/java/net/consensys/orion/config/Config.java @@ -29,10 +29,14 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import javax.annotation.Nullable; +import com.google.common.annotations.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.tuweni.config.Configuration; import org.apache.tuweni.config.ConfigurationError; import org.apache.tuweni.config.DocumentPosition; @@ -45,21 +49,33 @@ */ public class Config { + private static final Logger log = LogManager.getLogger(Config.class); + private static final Schema SCHEMA = configSchema(); public static Config load(final Path configFile) throws IOException { - return load(Configuration.fromToml(configFile, SCHEMA)); + return load(Configuration.fromToml(configFile, SCHEMA), System.getenv()); + } + + public static Config load(final String config, final Map env) { + return load(Configuration.fromToml(config, SCHEMA), env); } + @VisibleForTesting public static Config load(final String config) { - return load(Configuration.fromToml(config, SCHEMA)); + return load(Configuration.fromToml(config, SCHEMA), Collections.emptyMap()); } public static Config load(final InputStream is) throws IOException { - return load(Configuration.fromToml(is, SCHEMA)); + return load(is, System.getenv()); } - private static Config load(final Configuration configuration) { + @VisibleForTesting + public static Config load(final InputStream is, Map env) throws IOException { + return load(Configuration.fromToml(is, SCHEMA), env); + } + + private static Config load(final Configuration configuration, Map env) { final List errors = configuration.errors(); if (!errors.isEmpty()) { String errorString = errors.stream().limit(5).map(ConfigurationError::toString).collect(Collectors.joining("\n")); @@ -68,19 +84,21 @@ private static Config load(final Configuration configuration) { } throw new ConfigException(errorString); } - return new Config(configuration); + return new Config(configuration, env); } public static Config defaultConfig() { - return new Config(Configuration.empty(SCHEMA)); + return new Config(Configuration.empty(SCHEMA), System.getenv()); } private final Configuration configuration; private final Path workDir; + private final Map env; - private Config(final Configuration configuration) { + private Config(final Configuration configuration, final Map env) { this.configuration = configuration; - this.workDir = Paths.get(configuration.getString("workdir")); + this.env = env; + this.workDir = Paths.get(getString("workdir")); } /** @@ -101,6 +119,14 @@ public Optional nodeUrl() { * @return Port to listen on for the Orion API */ public int nodePort() { + String port = env.get(envKey("nodeport")); + if (port != null) { + try { + return Integer.parseInt(port); + } catch (NumberFormatException e) { + log.warn(e.getMessage(), e); + } + } return configuration.getInteger("nodeport"); } @@ -110,7 +136,7 @@ public int nodePort() { * @return the network interface to bind the Orion API to */ public String nodeNetworkInterface() { - return configuration.getString("nodenetworkinterface"); + return getString("nodenetworkinterface"); } /** @@ -131,6 +157,14 @@ public Optional clientUrl() { * @return Port to listen on for the client API */ public int clientPort() { + String port = env.get(envKey("clientport")); + if (port != null) { + try { + return Integer.parseInt(port); + } catch (NumberFormatException e) { + log.warn(e.getMessage(), e); + } + } return configuration.getInteger("clientport"); } @@ -140,7 +174,7 @@ public int clientPort() { * @return the network interface to bind the client API to. */ public String clientNetworkInterface() { - return configuration.getString("clientnetworkinterface"); + return getString("clientnetworkinterface"); } /** @@ -150,10 +184,10 @@ public String clientNetworkInterface() { */ @Nullable public Path libSodiumPath() { - if (!configuration.contains("libsodiumpath")) { - return null; + if (contains("libsodiumpath")) { + return Paths.get(getString("libsodiumpath")); } - return Paths.get(configuration.getString("libsodiumpath")); + return null; } /** @@ -178,6 +212,15 @@ public Path workDir() { * @return A list of other node URLs to connect to on startup. */ public List otherNodes() { + String otherNodes = env.get(envKey("othernodes")); + if (otherNodes != null) { + try { + return Arrays.stream(otherNodes.split(",")).map(URI::create).collect(Collectors.toList()); + } catch (IllegalArgumentException e) { + log.warn("Invalid ORION_OTHERNODES entry", e); + } + } + return configuration.getListOfString("othernodes").stream().map(urlString -> { try { return URI.create(urlString); @@ -236,10 +279,10 @@ public List alwaysSendTo() { * @see #privateKeys() */ public Optional passwords() { - if (!configuration.contains("passwords")) { - return Optional.empty(); + if (contains("passwords")) { + return Optional.of(getString("passwords")).map(workDir::resolve); } - return Optional.of(configuration.getString("passwords")).map(workDir::resolve); + return Optional.empty(); } /** @@ -257,7 +300,7 @@ public Optional passwords() { * @return Storage string specifying a storage engine and/or storage path */ public String storage() { - return System.getenv().getOrDefault("ORION_STORAGE", configuration.getString("storage")); + return getString("storage"); } /** @@ -275,7 +318,7 @@ public String storage() { * @return Storage string specifying a storage engine and/or storage path */ public String knownNodesStorage() { - return System.getenv().getOrDefault("ORION_KNOWN_NODE_STORAGE", configuration.getString("knownnodesstorage")); + return getString("knownnodesstorage"); } /** @@ -296,7 +339,7 @@ public String knownNodesStorage() { * @see #tlsClientTrust() */ public String tls() { - return configuration.getString("tls").toLowerCase(); + return getString("tls").toLowerCase(); } /** @@ -365,7 +408,7 @@ public Path tlsServerKey() { * @see #tlsKnownClients() */ public String tlsServerTrust() { - return configuration.getString("tlsservertrust").toLowerCase(); + return getString("tlsservertrust").toLowerCase(); } /** @@ -445,7 +488,7 @@ public Path tlsClientKey() { * @see #tlsKnownServers() */ public String tlsClientTrust() { - return configuration.getString("tlsclienttrust").toLowerCase(); + return getString("tlsclienttrust").toLowerCase(); } /** @@ -463,7 +506,7 @@ public Path tlsKnownServers() { } public String clientConnectionTls() { - return configuration.getString("clientconnectiontls").toLowerCase(); + return getString("clientconnectiontls").toLowerCase(); } public Path clientConnectionTlsServerCert() { @@ -479,29 +522,45 @@ public Path clientConnectionTlsServerKey() { } public String clientConnectionTlsServerTrust() { - return configuration.getString("clientconnectiontlsservertrust").toLowerCase(); + return getString("clientconnectiontlsservertrust").toLowerCase(); } public Path clientConnectionTlsKnownClients() { return getPath("clientconnectiontlsknownclients"); } + private String envKey(final String key) { + return "ORION_" + key.toUpperCase(); + } + + private boolean contains(final String key) { + return env.containsKey(envKey(key)) || configuration.contains(key); + } + + private String getString(final String key) { + return env.getOrDefault(envKey(key), configuration.getString(key)); + } + private Optional getURL(final String key) { try { - if (!configuration.contains(key)) { + if (!contains(key)) { return Optional.empty(); } - return Optional.of(new URL(configuration.getString(key))); + return Optional.of(new URL(getString(key))); } catch (final MalformedURLException e) { throw new IllegalStateException("key '" + key + "' should have been validated, yet it's invalid", e); } } private Path getPath(final String key) { - return workDir.resolve(configuration.getString(key)); + return workDir.resolve(getString(key)); } private List getListOfPath(final String key) { + String value = env.get(envKey(key)); + if (value != null) { + return Arrays.stream(value.split(",")).map(workDir::resolve).collect(Collectors.toList()); + } return configuration.getListOfString(key).stream().map(workDir::resolve).collect(Collectors.toList()); } diff --git a/src/test/java/net/consensys/orion/config/TomlConfigTest.java b/src/test/java/net/consensys/orion/config/TomlConfigTest.java index d87d1407..ea50accb 100644 --- a/src/test/java/net/consensys/orion/config/TomlConfigTest.java +++ b/src/test/java/net/consensys/orion/config/TomlConfigTest.java @@ -24,6 +24,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import org.junit.jupiter.api.Test; @@ -32,13 +34,14 @@ class TomlConfigTest { @Test void fullFileRead() throws Exception { final InputStream configAsStream = this.getClass().getClassLoader().getResourceAsStream("fullConfigTest.toml"); - final Config testConf = Config.load(configAsStream); + final Config testConf = Config.load(configAsStream, Collections.emptyMap()); final URL expectedURL = new URL("http://127.0.0.1:9001/"); assertEquals(expectedURL, testConf.nodeUrl().get()); assertEquals(9001, testConf.nodePort()); assertEquals("0.0.0.0", testConf.nodeNetworkInterface()); assertEquals("memory", testConf.storage()); + assertEquals("memory", testConf.knownNodesStorage()); assertEquals("off", testConf.tls()); assertEquals("ca-or-tofu", testConf.tlsServerTrust()); assertEquals("ca", testConf.tlsClientTrust()); @@ -76,11 +79,37 @@ void fullFileRead() throws Exception { assertEquals(Collections.singletonList(URI.create("http://127.0.0.1:9000/")), testConf.otherNodes()); } + @Test + void testEnvOverride() throws Exception { + final InputStream configAsStream = this.getClass().getClassLoader().getResourceAsStream("fullConfigTest.toml"); + Map overrides = new HashMap<>(); + overrides.put("ORION_NODEPORT", "10001"); + overrides.put("ORION_NODENETWORKINTERFACE", "192.168.0.1"); + overrides.put("ORION_NODEURL", "http://192.168.0.1:10001/"); + overrides.put("ORION_STORAGE", "mapdb:somefolder"); + overrides.put("ORION_KNOWNNODESSTORAGE", "mapdb:someotherfolder"); + overrides.put("ORION_TLS", "on"); + overrides.put("ORION_TLSSERVERTRUST", "ca"); + final Config testConf = Config.load(configAsStream, overrides); + + final URL expectedURL = new URL("http://192.168.0.1:10001/"); + assertEquals(expectedURL, testConf.nodeUrl().get()); + assertEquals(10001, testConf.nodePort()); + assertEquals("192.168.0.1", testConf.nodeNetworkInterface()); + assertEquals("mapdb:somefolder", testConf.storage()); + assertEquals("mapdb:someotherfolder", testConf.knownNodesStorage()); + + assertEquals("on", testConf.tls()); + assertEquals("ca", testConf.tlsServerTrust()); + } + @Test void fullFileReadUsingDefaults() throws Exception { - final Config testConf = Config.load(this.getClass().getClassLoader().getResourceAsStream("defaultConfigTest.toml")); + final Config testConf = Config + .load(this.getClass().getClassLoader().getResourceAsStream("defaultConfigTest.toml"), Collections.emptyMap()); assertEquals("leveldb", testConf.storage()); + assertEquals("memory", testConf.knownNodesStorage()); assertEquals("off", testConf.tls()); assertEquals("tofu", testConf.tlsServerTrust()); assertEquals("ca-or-tofu", testConf.tlsClientTrust()); diff --git a/src/test/resources/fullConfigTest.toml b/src/test/resources/fullConfigTest.toml index 9bca3a92..03f999a1 100644 --- a/src/test/resources/fullConfigTest.toml +++ b/src/test/resources/fullConfigTest.toml @@ -1,4 +1,5 @@ # Test file with complete config settings (not defaults) +knownnodesstorage = "memory" nodeurl = "http://127.0.0.1:9001/" nodeport = 9001 nodenetworkinterface = "0.0.0.0" From c9cde272d40ac437a691f265ab2f74b429a9c8cc Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Thu, 20 Feb 2020 23:15:14 -0800 Subject: [PATCH 18/19] add test checking list of things --- src/test/java/net/consensys/orion/config/TomlConfigTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/net/consensys/orion/config/TomlConfigTest.java b/src/test/java/net/consensys/orion/config/TomlConfigTest.java index ea50accb..0323fe3d 100644 --- a/src/test/java/net/consensys/orion/config/TomlConfigTest.java +++ b/src/test/java/net/consensys/orion/config/TomlConfigTest.java @@ -23,6 +23,7 @@ import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -88,6 +89,7 @@ void testEnvOverride() throws Exception { overrides.put("ORION_NODEURL", "http://192.168.0.1:10001/"); overrides.put("ORION_STORAGE", "mapdb:somefolder"); overrides.put("ORION_KNOWNNODESSTORAGE", "mapdb:someotherfolder"); + overrides.put("ORION_OTHERNODES", "foo,bar,noes"); overrides.put("ORION_TLS", "on"); overrides.put("ORION_TLSSERVERTRUST", "ca"); final Config testConf = Config.load(configAsStream, overrides); @@ -98,6 +100,7 @@ void testEnvOverride() throws Exception { assertEquals("192.168.0.1", testConf.nodeNetworkInterface()); assertEquals("mapdb:somefolder", testConf.storage()); assertEquals("mapdb:someotherfolder", testConf.knownNodesStorage()); + assertEquals(Arrays.asList(URI.create("foo"), URI.create("bar"), URI.create("noes")), testConf.otherNodes()); assertEquals("on", testConf.tls()); assertEquals("ca", testConf.tlsServerTrust()); From e691f7c5b1110110fa02f6f1c85a972d3906d218 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Sun, 23 Feb 2020 18:27:52 -0800 Subject: [PATCH 19/19] refactor integer logic to getInteger --- .../net/consensys/orion/config/Config.java | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/main/java/net/consensys/orion/config/Config.java b/src/main/java/net/consensys/orion/config/Config.java index bb1ad51c..a8116371 100644 --- a/src/main/java/net/consensys/orion/config/Config.java +++ b/src/main/java/net/consensys/orion/config/Config.java @@ -119,15 +119,7 @@ public Optional nodeUrl() { * @return Port to listen on for the Orion API */ public int nodePort() { - String port = env.get(envKey("nodeport")); - if (port != null) { - try { - return Integer.parseInt(port); - } catch (NumberFormatException e) { - log.warn(e.getMessage(), e); - } - } - return configuration.getInteger("nodeport"); + return getInteger("nodeport"); } /** @@ -157,15 +149,7 @@ public Optional clientUrl() { * @return Port to listen on for the client API */ public int clientPort() { - String port = env.get(envKey("clientport")); - if (port != null) { - try { - return Integer.parseInt(port); - } catch (NumberFormatException e) { - log.warn(e.getMessage(), e); - } - } - return configuration.getInteger("clientport"); + return getInteger("clientport"); } /** @@ -537,6 +521,18 @@ private boolean contains(final String key) { return env.containsKey(envKey(key)) || configuration.contains(key); } + private Integer getInteger(final String key) { + String valueStr = env.get(envKey(key)); + if (valueStr != null) { + try { + return Integer.parseInt(valueStr); + } catch (NumberFormatException e) { + log.warn(e.getMessage(), e); + } + } + return configuration.getInteger(key); + } + private String getString(final String key) { return env.getOrDefault(envKey(key), configuration.getString(key)); }