Add a configurable memory limit to the torrent repository and implement dashmap [#567, #565]

Cargo.toml

@@ -38,9 +38,12 @@ axum-server = { version = "0", features = ["tls-rustls"] }
 binascii = "0"
 chrono = { version = "0", default-features = false, features = ["clock"] }
 config = "0"
+dashmap = { version = "5.5.3", features = ["raw-api"] }
+deepsize = "0.2.0"
 derive_more = "0"
 fern = "0"
 futures = "0"
+hashbrown = "0"
 hyper = "1"
 lazy_static = "1"
 log = { version = "0", features = ["release_max_level_info"] }

packages/configuration/src/lib.rs

@@ -431,6 +431,10 @@ pub struct Configuration {
     /// Tracker mode. See [`TrackerMode`] for more information.
     pub mode: TrackerMode,
 
+    /// (Optional) Set the max amount of space the torrent repository can use for storing torrents.
+    /// Size should be in in MegaBytes.
+    pub max_torrent_repository_size: Option<usize>,
+
     // Database configuration
     /// Database driver. Possible values are: `Sqlite3`, and `MySQL`.
     pub db_driver: DatabaseDriver,
@@ -536,6 +540,7 @@ impl Default for Configuration {
         let mut configuration = Configuration {
             log_level: Option::from(String::from("info")),
             mode: TrackerMode::Public,
+            max_torrent_repository_size: None,
             db_driver: DatabaseDriver::Sqlite3,
             db_path: String::from("./storage/tracker/lib/database/sqlite3.db"),
             announce_interval: announce_policy.interval,

packages/torrent-repository-benchmarks/src/benches/asyn.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 
 use clap::Parser;
 use futures::stream::FuturesUnordered;
-use torrust_tracker::core::torrent::repository::TRepositoryAsync;
+use torrust_tracker::core::torrent::repositories::TRepositoryAsync;
 use torrust_tracker::shared::bit_torrent::info_hash::InfoHash;
 
 use crate::args::Args;
@@ -20,7 +20,7 @@ pub async fn async_add_one_torrent<T: TRepositoryAsync + Send + Sync + 'static>(
         let start_time = std::time::Instant::now();
 
         torrent_repository
-            .update_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER)
+            .upsert_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER)
             .await;
 
         let result = start_time.elapsed();
@@ -46,7 +46,7 @@ pub async fn async_update_one_torrent_in_parallel<T: TRepositoryAsync + Send + S
 
         // Add the torrent/peer to the torrent repository
         torrent_repository
-            .update_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER)
+            .upsert_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER)
             .await;
 
         let start_time = std::time::Instant::now();
@@ -56,7 +56,7 @@ pub async fn async_update_one_torrent_in_parallel<T: TRepositoryAsync + Send + S
 
             let handle = runtime.spawn(async move {
                 torrent_repository_clone
-                    .update_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER)
+                    .upsert_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER)
                     .await;
 
                 if let Some(sleep_time) = args.sleep {
@@ -100,7 +100,7 @@ pub async fn async_add_multiple_torrents_in_parallel<T: TRepositoryAsync + Send
 
             let handle = runtime.spawn(async move {
                 torrent_repository_clone
-                    .update_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER)
+                    .upsert_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER)
                     .await;
 
                 if let Some(sleep_time) = args.sleep {
@@ -140,7 +140,7 @@ pub async fn async_update_multiple_torrents_in_parallel<T: TRepositoryAsync + Se
         // Add the torrents/peers to the torrent repository
         for info_hash in &info_hashes {
             torrent_repository
-                .update_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER)
+                .upsert_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER)
                 .await;
         }
 
@@ -151,7 +151,7 @@ pub async fn async_update_multiple_torrents_in_parallel<T: TRepositoryAsync + Se
 
             let handle = runtime.spawn(async move {
                 torrent_repository_clone
-                    .update_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER)
+                    .upsert_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER)
                     .await;
 
                 if let Some(sleep_time) = args.sleep {

packages/torrent-repository-benchmarks/src/benches/sync.rs

@@ -3,7 +3,7 @@ use std::time::Duration;
 
 use clap::Parser;
 use futures::stream::FuturesUnordered;
-use torrust_tracker::core::torrent::repository::Repository;
+use torrust_tracker::core::torrent::repositories::Repository;
 use torrust_tracker::shared::bit_torrent::info_hash::InfoHash;
 
 use crate::args::Args;
@@ -21,7 +21,7 @@ pub fn add_one_torrent<T: Repository + Send + Sync + 'static>(samples: usize) ->
 
         let start_time = std::time::Instant::now();
 
-        torrent_repository.update_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER);
+        torrent_repository.upsert_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER);
 
         let result = start_time.elapsed();
 
@@ -45,15 +45,15 @@ pub async fn update_one_torrent_in_parallel<T: Repository + Send + Sync + 'stati
         let handles = FuturesUnordered::new();
 
         // Add the torrent/peer to the torrent repository
-        torrent_repository.update_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER);
+        torrent_repository.upsert_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER);
 
         let start_time = std::time::Instant::now();
 
         for _ in 0..10_000 {
             let torrent_repository_clone = torrent_repository.clone();
 
             let handle = runtime.spawn(async move {
-                torrent_repository_clone.update_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER);
+                torrent_repository_clone.upsert_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER);
 
                 if let Some(sleep_time) = args.sleep {
                     let start_time = std::time::Instant::now();
@@ -95,7 +95,7 @@ pub async fn add_multiple_torrents_in_parallel<T: Repository + Send + Sync + 'st
             let torrent_repository_clone = torrent_repository.clone();
 
             let handle = runtime.spawn(async move {
-                torrent_repository_clone.update_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER);
+                torrent_repository_clone.upsert_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER);
 
                 if let Some(sleep_time) = args.sleep {
                     let start_time = std::time::Instant::now();
@@ -133,7 +133,7 @@ pub async fn update_multiple_torrents_in_parallel<T: Repository + Send + Sync +
 
         // Add the torrents/peers to the torrent repository
         for info_hash in &info_hashes {
-            torrent_repository.update_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER);
+            torrent_repository.upsert_torrent_with_peer_and_get_stats(info_hash, &DEFAULT_PEER);
         }
 
         let start_time = std::time::Instant::now();
@@ -142,7 +142,7 @@ pub async fn update_multiple_torrents_in_parallel<T: Repository + Send + Sync +
             let torrent_repository_clone = torrent_repository.clone();
 
             let handle = runtime.spawn(async move {
-                torrent_repository_clone.update_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER);
+                torrent_repository_clone.upsert_torrent_with_peer_and_get_stats(&info_hash, &DEFAULT_PEER);
 
                 if let Some(sleep_time) = args.sleep {
                     let start_time = std::time::Instant::now();

packages/torrent-repository-benchmarks/src/main.rs

@@ -7,7 +7,9 @@ use torrust_torrent_repository_benchmarks::benches::asyn::{
 use torrust_torrent_repository_benchmarks::benches::sync::{
     add_multiple_torrents_in_parallel, add_one_torrent, update_multiple_torrents_in_parallel, update_one_torrent_in_parallel,
 };
-use torrust_tracker::core::torrent::repository::{AsyncSync, RepositoryAsync, RepositoryAsyncSingle, Sync, SyncSingle};
+use torrust_tracker::core::torrent::repositories::{
+    AsyncSync, RepositoryAsync, RepositoryAsyncSingle, RepositoryDashmap, Sync, SyncSingle,
+};
 
 #[allow(clippy::too_many_lines)]
 #[allow(clippy::print_literal)]
@@ -135,5 +137,29 @@ fn main() {
             "update_multiple_torrents_in_parallel",
             rt.block_on(async_update_multiple_torrents_in_parallel::<RepositoryAsync>(&rt, 10))
         );
+
+        println!();
+
+        println!("DashMap<InfoHash, Entry>");
+        println!(
+            "{}: Avg/AdjAvg: {:?}",
+            "add_one_torrent",
+            add_one_torrent::<RepositoryDashmap>(1_000_000)
+        );
+        println!(
+            "{}: Avg/AdjAvg: {:?}",
+            "update_one_torrent_in_parallel",
+            rt.block_on(update_one_torrent_in_parallel::<RepositoryDashmap>(&rt, 10))
+        );
+        println!(
+            "{}: Avg/AdjAvg: {:?}",
+            "add_multiple_torrents_in_parallel",
+            rt.block_on(add_multiple_torrents_in_parallel::<RepositoryDashmap>(&rt, 10))
+        );
+        println!(
+            "{}: Avg/AdjAvg: {:?}",
+            "update_multiple_torrents_in_parallel",
+            rt.block_on(update_multiple_torrents_in_parallel::<RepositoryDashmap>(&rt, 10))
+        );
     }
 }