a

Author: cassanof

get_metadata.py

@@ -1,8 +1,6 @@
 import datasets
 import datetime
 import json
-from tqdm import tqdm
-import os
 ds = datasets.load_dataset("CCRss/arXiv_dataset", split="train")
 
 # only keep "id"

process_dataset.py

@@ -0,0 +1,20 @@
+import datasets
+import os
+import datetime
+from pathlib import Path
+from tqdm import tqdm
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--papers", type=str, required=True)
+parser.add_argument("--push", type=str, required=True)
+args = parser.parse_args()
+
+ds = datasets.load_dataset("CCRss/arXiv_dataset", split="train")
+
+
+def process(ex):
+    return ex
+
+
+ds = ds.map(process, num_proc=os.cpu_count())

src/main.rs

@@ -86,7 +86,7 @@ async fn download_paper(id: &str) -> Result<Paper, Box<dyn Error>> {
         let response = reqwest::get(&url).await?;
         // check status code, if 429, wait and retry
         if response.status().as_u16() == 429 {
-            println!("429: Waiting 120 seconds");
+            println!("429: Waiting {} seconds", backoff);
             tokio::time::sleep(std::time::Duration::from_secs(backoff)).await;
             backoff *= 2;
             continue;