refactor

Makefile

@@ -39,9 +39,9 @@ init: $(WASTE_REPORT) ## Clone output repositories for report generation. Only n
 fetch-only: $(RELEASE_EXECUTABLE) $(bare_index_path) ## Run the fetch stage once
 		$(RELEASE_EXECUTABLE) mine -c $(bare_index_path) -F 1 -P 0 -R 0 $(DB)
 process-only: $(RELEASE_EXECUTABLE) $(bare_index_path) ## Run the processing stage once
-		$(RELEASE_EXECUTABLE) mine -c $(bare_index_path) --io 1 --cpu 2  -F 0 -P 1 -R 0 $(DB)
+		$(RELEASE_EXECUTABLE) mine -c $(bare_index_path) --io 10 --cpu 2  -F 0 -P 1 -R 0 $(DB)
 process-only-nonstop: $(RELEASE_EXECUTABLE) $(bare_index_path) ## Run the processing stage continuously
-		$(RELEASE_EXECUTABLE) mine -c $(bare_index_path) --io 1 --cpu 2  -F 0 -p 5min -R 0 $(DB)
+		$(RELEASE_EXECUTABLE) mine -c $(bare_index_path) --io 10 --cpu 2  -F 0 -p 5min -R 0 $(DB)
 report-only: $(RELEASE_EXECUTABLE) $(bare_index_path) ## Run the reporting stage once
 		$(RELEASE_EXECUTABLE) mine -c $(bare_index_path) --cpu-o 10  -F 0 -P 0 -R 1 $(DB)
 force-report-only: $(RELEASE_EXECUTABLE) $(bare_index_path) ## Run the reporting stage once, forcibly, rewriting everything and ignoring caches

criner/README.md

@@ -2,7 +2,6 @@
 
 ## TODO
 
-* [ ] Set 30s/10s timeouts for chunk awaits and head awaits (on top of total timeout set in reqwest). That should make it way more reliable
 * [ ] Make use of crates-io database easy and integrate with criner to allow using download counts and other meta-data, like release date
   * [ ] Consider extracting the release date from git and upgrade `crates-index-diff` to support that.
 * [ ] See why RipGrep doesn't get any suggestions

criner/src/engine/work/iobound.rs

@@ -15,6 +15,9 @@ use async_trait::async_trait;
 use futures::FutureExt;
 use std::time::Duration;
 
+const CONNECT_AND_FETCH_HEAD_TIMEOUT: Duration = Duration::from_secs(30);
+const FETCH_CHUNK_TIMEOUT_SECONDS: Duration = Duration::from_secs(20);
+
 struct ProcessingState {
     url: String,
     kind: &'static str,
@@ -176,7 +179,7 @@ async fn download_file_and_store_result(
 ) -> Result<()> {
     progress.blocked("fetch HEAD", None);
     let mut res = timeout_after(
-        Duration::from_secs(30),
+        CONNECT_AND_FETCH_HEAD_TIMEOUT,
         "fetching HEAD",
         client.get(url).send(),
     )
@@ -201,7 +204,7 @@ async fn download_file_and_store_result(
         .await?;
 
     while let Some(chunk) = timeout_after(
-        Duration::from_secs(15),
+        FETCH_CHUNK_TIMEOUT_SECONDS,
         format!(
             "fetching {} of {}",
             ByteSize(bytes_received as u64),