Feature/network monitor (#369)

* WIP commit

* Further into errors (WIP)

* WIP commit, more stuff compiling, but now we switch to sneaky Jedrzej trick

* Websocket connections starting to work

* WIP

* Constructing the socket stream in one go

* Nicer connections. Getting topology now works.

* Fixing startup message

* Injecting directory uri

* Injecting good mixndodes

* Deleting old healthcheck

* Starting to wrap gateway client

* Splitting out good topology contstruction

* Starting to breathe finally

* Fixed layer assignment error

* Starting gateway client in a better spot

* Cleanup

* Renamed construct() to new()

* Injected channels, removed websockets

* Added a MixnetListener to deal with returned packet traffic

* Simplification

* Renaming and commenting a few things

* Renamed temp variables

* Fixed variable names

* Made errors a bit more explicit on message reconstruction

* Added the mixmining route / object to the directory server client

* Recipient is always "me", a bit easier to understand what's going on

* Subbing in test nodes works

* Using QA directory

* Noting which directory server we're using at monitor start

* Adding Debug to MixStatus so we can print it more easily

* Prettification of startup messages

* Notifying directory of successful packet receipt

* All nodes now being tested

* Now able to do timer-based test runs and detect the last node tested

* Renamed mixnet_listener to notifier

* Moved message chunking into its own home

* Starting to pull out packet sends

* Basic functionality working!

* Enabled timer runs forever

* License notices

* Depdendency cleanup

* Import cleanup

* Moved to implicit tokio runtime

* Removed some unnecessary data clones

* Fixed monitor doing two runs at startup rather than one

* dalek version updates

* This should have been split but got carried away

* Initial ipv6 support

* Don't test outdated nodes

* Added network monitor to default workspace members

* Actually sending reports

* Batch-sending mix status

* Cargo fmt

* Minor cleanup

* Malformed Cargo.lock

* Post-merge fixes

* Defined constant in main.rs for enabling detailed reporting

* Updated package versions of the system

* Checking version compatibility on startup

Co-authored-by: Dave Hrycyszyn <futurechimp@users.noreply.github.com>

Author: jstuczyn

Cargo.lock

@@ -41,6 +41,7 @@ members = [
     "gateway/gateway-requests",
     "service-providers/sphinx-socks",
     "mixnode",
+    "network-monitor",
     "validator",
 ]
 
@@ -51,4 +52,5 @@ default-members = [
     "gateway",
     "mixnode",
     "validator",
+    "network-monitor",
 ]

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 build = "build.rs"
 name = "nym-client"
-version = "0.8.1"
+version = "0.9.0-dev"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>", "Jędrzej Stuczyński <andrew@nymtech.net>"]
 edition = "2018"
 

clients/native/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 build = "build.rs"
 name = "nym-socks5-client"
-version = "0.8.1"
+version = "0.9.0-dev"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>"]
 edition = "2018"
 

clients/socks5/Cargo.toml

@@ -2,7 +2,7 @@
 build = "build.rs"
 name = "nym-client-wasm"
 authors = ["Dave Hrycyszyn <futurechimp@users.noreply.github.com>", "Jedrzej Stuczynski <andrew@nymtech.net>"]
-version = "0.8.1"
+version = "0.9.0-dev"
 edition = "2018"
 keywords = ["nym", "sphinx", "wasm", "webassembly", "privacy", "client"]
 license = "Apache-2.0"

clients/webassembly/Cargo.toml

@@ -13,4 +13,5 @@
 // limitations under the License.
 
 pub mod metrics;
+pub mod mixmining;
 pub mod presence;

common/client-libs/directory-client/models/src/lib.rs

@@ -0,0 +1,19 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+/// A notification sent to the validators to let them know whether a given mix is
+/// currently up or down (based on whether it's mixing packets)
+pub struct MixStatus {
+    pub pub_key: String,
+    pub ip_version: String,
+    pub up: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
+/// A notification sent to the validators to let them know whether a given set of mixes is
+/// currently up or down (based on whether it's mixing packets)
+pub struct BatchMixStatus {
+    pub status: Vec<MixStatus>,
+}

common/client-libs/directory-client/models/src/mixmining.rs

@@ -36,7 +36,7 @@ impl From<io::Error> for ConversionError {
     }
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct MixNodePresence {
     pub location: String,

common/client-libs/directory-client/models/src/presence/mixnodes.rs

@@ -15,6 +15,8 @@
 use crate::requests::health_check_get::Request as HealthCheckRequest;
 use crate::requests::metrics_mixes_get::Request as MetricsMixRequest;
 use crate::requests::metrics_mixes_post::Request as MetricsMixPost;
+use crate::requests::mix_mining_batch_status_post::Request as MixMiningBatchStatusPost;
+use crate::requests::mix_mining_status_post::Request as MixMiningStatusPost;
 use crate::requests::presence_coconodes_post::Request as PresenceCocoNodesPost;
 use crate::requests::presence_gateways_post::Request as PresenceGatewayPost;
 use crate::requests::presence_mixnodes_post::Request as PresenceMixNodesPost;
@@ -25,10 +27,12 @@ use directory_client_models::presence::{
     coconodes::CocoPresence, gateways::GatewayPresence, mixnodes::MixNodePresence,
     providers::MixProviderPresence,
 };
+use mixmining::MixStatus;
 use requests::{health_check_get::HealthCheckResponse, DirectoryGetRequest, DirectoryPostRequest};
 
+use directory_client_models::mixmining::BatchMixStatus;
 pub use directory_client_models::{
-    metrics,
+    metrics, mixmining,
     presence::{self, Topology},
 };
 
@@ -123,6 +127,22 @@ impl Client {
         self.post(req).await
     }
 
+    pub async fn post_mixmining_status(
+        &self,
+        status: MixStatus,
+    ) -> reqwest::Result<reqwest::Response> {
+        let req = MixMiningStatusPost::new(&self.base_url, status);
+        self.post(req).await
+    }
+
+    pub async fn post_batch_mixmining_status(
+        &self,
+        batch_status: BatchMixStatus,
+    ) -> reqwest::Result<reqwest::Response> {
+        let req = MixMiningBatchStatusPost::new(&self.base_url, batch_status);
+        self.post(req).await
+    }
+
     // this should be soft-deprecated as the whole concept of provider will
     // be removed in the next topology rework
     pub async fn post_provider_presence(

common/client-libs/directory-client/src/lib.rs

@@ -0,0 +1,91 @@
+use super::{DirectoryPostRequest, DirectoryRequest};
+use crate::mixmining::BatchMixStatus;
+
+const PATH: &str = "/api/mixmining/batch";
+
+pub struct Request {
+    base_url: String,
+    path: String,
+    payload: BatchMixStatus,
+}
+
+impl DirectoryRequest for Request {
+    fn url(&self) -> String {
+        format!("{}{}", self.base_url, self.path)
+    }
+}
+
+impl DirectoryPostRequest for Request {
+    type Payload = BatchMixStatus;
+    fn new(base_url: &str, payload: Self::Payload) -> Self {
+        Request {
+            base_url: base_url.to_string(),
+            path: PATH.to_string(),
+            payload,
+        }
+    }
+
+    fn json_payload(&self) -> &BatchMixStatus {
+        &self.payload
+    }
+}
+
+#[cfg(test)]
+mod batch_mix_status_post_request {
+    use super::*;
+    use crate::client_test_fixture;
+    use mockito::mock;
+
+    #[cfg(test)]
+    mod on_a_400_status {
+        use super::*;
+
+        #[tokio::test]
+        async fn it_returns_an_error() {
+            let _m = mock("POST", PATH).with_status(400).create();
+            let client = client_test_fixture(&mockito::server_url());
+            let result = client
+                .post_batch_mixmining_status(fixtures::new_status())
+                .await;
+            assert_eq!(400, result.unwrap().status());
+            _m.assert();
+        }
+    }
+
+    #[cfg(test)]
+    mod on_a_201 {
+        use super::*;
+
+        #[tokio::test]
+        async fn it_returns_a_response_with_201() {
+            let json = r#"{
+                "ok": true
+            }"#;
+            let _m = mock("POST", "/api/mixmining/batch")
+                .with_status(201)
+                .with_body(json)
+                .create();
+            let client = client_test_fixture(&mockito::server_url());
+            let result = client
+                .post_batch_mixmining_status(fixtures::new_status())
+                .await;
+            assert!(result.is_ok());
+            _m.assert();
+        }
+    }
+
+    #[cfg(test)]
+    mod fixtures {
+        use crate::mixmining::{BatchMixStatus, MixStatus};
+
+        pub fn new_status() -> BatchMixStatus {
+            BatchMixStatus {
+                status: vec![MixStatus {
+                    pub_key: "abc".to_string(),
+                    ip_version: "4".to_string(),
+                    up: true,
+                }],
+            }
+        }
+    }
+}