feat(cli): support auth tokens for accessing private modules

Closes denoland#5239
kitsonk · Feb 14, 2021 · f144c9f · f144c9f
1 parent 5873ade
commit f144c9f
Show file tree

Hide file tree

Showing 9 changed files with 284 additions and 18 deletions.
diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
@@ -4,6 +4,7 @@ use crate::colors;
 use crate::http_cache::HttpCache;
 use crate::http_util::create_http_client;
 use crate::http_util::fetch_once;
+use crate::http_util::AuthTokens;
 use crate::http_util::FetchOnceResult;
 use crate::media_type::MediaType;
 use crate::text_encoding;
@@ -19,6 +20,7 @@ use deno_core::futures::future::FutureExt;
 use deno_core::ModuleSpecifier;
 use deno_runtime::deno_fetch::reqwest;
 use std::collections::HashMap;
+use std::env;
 use std::fs;
 use std::future::Future;
 use std::io::Read;
@@ -27,6 +29,7 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::sync::Mutex;
 
+static DENO_AUTH_TOKENS: &str = "DENO_AUTH_TOKENS";
 pub const SUPPORTED_SCHEMES: [&str; 4] = ["data", "file", "http", "https"];
 
 /// A structure representing a source file.
@@ -308,6 +311,7 @@ fn strip_shebang(mut value: String) -> String {
 /// A structure for resolving, fetching and caching source files.
 #[derive(Clone)]
 pub struct FileFetcher {
+  auth_tokens: AuthTokens,
   allow_remote: bool,
   cache: FileCache,
   cache_setting: CacheSetting,
@@ -323,8 +327,9 @@ impl FileFetcher {
     ca_data: Option<Vec<u8>>,
   ) -> Result<Self, AnyError> {
     Ok(Self {
+      auth_tokens: AuthTokens::new(env::var(DENO_AUTH_TOKENS).ok()),
       allow_remote,
-      cache: FileCache::default(),
+      cache: Default::default(),
       cache_setting,
       http_cache,
       http_client: create_http_client(get_user_agent(), ca_data)?,
@@ -478,17 +483,25 @@ impl FileFetcher {
 
     info!("{} {}", colors::green("Download"), specifier);
 
-    let file_fetcher = self.clone();
-    let cached_etag = match self.http_cache.get(specifier.as_url()) {
+    let maybe_etag = match self.http_cache.get(specifier.as_url()) {
       Ok((_, headers)) => headers.get("etag").cloned(),
       _ => None,
     };
+    let maybe_auth_token = self.auth_tokens.get(&specifier);
     let specifier = specifier.clone();
     let permissions = permissions.clone();
     let http_client = self.http_client.clone();
+    let file_fetcher = self.clone();
     // A single pass of fetch either yields code or yields a redirect.
     async move {
-      match fetch_once(http_client, specifier.as_url(), cached_etag).await? {
+      match fetch_once(
+        http_client,
+        specifier.as_url(),
+        maybe_etag,
+        maybe_auth_token,
+      )
+      .await?
+      {
         FetchOnceResult::NotModified => {
           let file = file_fetcher.fetch_cached(&specifier, 10)?.unwrap();
           Ok(file)

diff --git a/cli/http_util.rs b/cli/http_util.rs
@@ -3,9 +3,11 @@
 use deno_core::error::generic_error;
 use deno_core::error::AnyError;
 use deno_core::url::Url;
+use deno_core::ModuleSpecifier;
 use deno_runtime::deno_fetch::reqwest;
 use deno_runtime::deno_fetch::reqwest::header::HeaderMap;
 use deno_runtime::deno_fetch::reqwest::header::HeaderValue;
+use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION;
 use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH;
 use deno_runtime::deno_fetch::reqwest::header::LOCATION;
 use deno_runtime::deno_fetch::reqwest::header::USER_AGENT;
@@ -14,6 +16,56 @@ use deno_runtime::deno_fetch::reqwest::Client;
 use deno_runtime::deno_fetch::reqwest::StatusCode;
 use std::collections::HashMap;
 
+#[derive(Debug, Clone)]
+struct AuthToken {
+  host: String,
+  token: String,
+}
+
+#[derive(Debug, Clone)]
+pub struct AuthTokens(Vec<AuthToken>);
+
+impl AuthTokens {
+  pub fn new(maybe_token_str: Option<String>) -> Self {
+    let mut tokens = Vec::new();
+    if let Some(tokens_str) = maybe_token_str {
+      for token_str in tokens_str.split(';') {
+        if token_str.contains('@') {
+          let pair: Vec<&str> = token_str.split('@').collect();
+          if pair.len() == 2 {
+            let token = pair[0].to_string();
+            let host = pair[1].to_lowercase();
+            tokens.push(AuthToken { host, token });
+          } else {
+            error!("Badly formed auth token discarded.");
+          }
+        } else {
+          error!("Badly formed auth token discarded.");
+        }
+      }
+      debug!("Parsed {} auth token(s).", tokens.len());
+    }
+
+    Self(tokens)
+  }
+
+  pub fn get(&self, specifier: &ModuleSpecifier) -> Option<String> {
+    self.0.iter().find_map(|t| {
+      let url = specifier.as_url();
+      let hostname = if let Some(port) = url.port() {
+        format!("{}:{}", url.host_str()?, port)
+      } else {
+        url.host_str()?.to_string()
+      };
+      if hostname.to_lowercase().ends_with(&t.host) {
+        Some(t.token.clone())
+      } else {
+        None
+      }
+    })
+  }
+}
+
 /// Create new instance of async reqwest::Client. This client supports
 /// proxies and doesn't follow redirects.
 pub fn create_http_client(
@@ -84,16 +136,23 @@ pub enum FetchOnceResult {
 pub async fn fetch_once(
   client: Client,
   url: &Url,
-  cached_etag: Option<String>,
+  maybe_etag: Option<String>,
+  maybe_auth_token: Option<String>,
 ) -> Result<FetchOnceResult, AnyError> {
   let url = url.clone();
 
   let mut request = client.get(url.clone());
 
-  if let Some(etag) = cached_etag {
+  if let Some(etag) = maybe_etag {
     let if_none_match_val = HeaderValue::from_str(&etag).unwrap();
     request = request.header(IF_NONE_MATCH, if_none_match_val);
   }
+  if let Some(auth_token) = maybe_auth_token {
+    let authorization_val_str = format!("Bearer {}", auth_token);
+    let authorization_val =
+      HeaderValue::from_str(&authorization_val_str).unwrap();
+    request = request.header(AUTHORIZATION, authorization_val);
+  }
   let response = request.send().await?;
 
   if response.status() == StatusCode::NOT_MODIFIED {
@@ -165,7 +224,7 @@ mod tests {
     let url =
       Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None, None).await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(headers.get("content-type").unwrap(), "application/json");
@@ -185,7 +244,7 @@ mod tests {
     )
     .unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None, None).await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");
       assert_eq!(
@@ -204,7 +263,7 @@ mod tests {
     let _http_server_guard = test_util::http_server();
     let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client.clone(), &url, None).await;
+    let result = fetch_once(client.clone(), &url, None, None).await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");
@@ -218,7 +277,8 @@ mod tests {
     }
 
     let res =
-      fetch_once(client, &url, Some("33a64df551425fcc55e".to_string())).await;
+      fetch_once(client, &url, Some("33a64df551425fcc55e".to_string()), None)
+        .await;
     assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
   }
 
@@ -231,7 +291,7 @@ mod tests {
     )
     .unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None, None).await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");
@@ -256,7 +316,7 @@ mod tests {
     let target_url =
       Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None, None).await;
     if let Ok(FetchOnceResult::Redirect(url, _)) = result {
       assert_eq!(url, target_url);
     } else {
@@ -322,7 +382,7 @@ mod tests {
       ),
     )
     .unwrap();
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None, None).await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(headers.get("content-type").unwrap(), "application/json");
@@ -354,7 +414,7 @@ mod tests {
       ),
     )
     .unwrap();
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None, None).await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");
       assert_eq!(
@@ -385,7 +445,7 @@ mod tests {
       ),
     )
     .unwrap();
-    let result = fetch_once(client.clone(), &url, None).await;
+    let result = fetch_once(client.clone(), &url, None, None).await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");
@@ -400,7 +460,8 @@ mod tests {
     }
 
     let res =
-      fetch_once(client, &url, Some("33a64df551425fcc55e".to_string())).await;
+      fetch_once(client, &url, Some("33a64df551425fcc55e".to_string()), None)
+        .await;
     assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
   }
 
@@ -425,7 +486,7 @@ mod tests {
       ),
     )
     .unwrap();
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None, None).await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");
@@ -446,10 +507,55 @@ mod tests {
     let url_str = "http://127.0.0.1:4545/bad_redirect";
     let url = Url::parse(url_str).unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(client, &url, None, None).await;
     assert!(result.is_err());
     let err = result.unwrap_err();
     // Check that the error message contains the original URL
     assert!(err.to_string().contains(url_str));
   }
+
+  #[test]
+  fn test_auth_token() {
+    let auth_tokens = AuthTokens::new(Some("abc123@deno.land".to_string()));
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land/x/mod.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), Some("abc123".to_string()));
+    let fixture =
+      ModuleSpecifier::resolve_url("https://www.deno.land/x/mod.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), Some("abc123".to_string()));
+    let fixture =
+      ModuleSpecifier::resolve_url("http://127.0.0.1:8080/x/mod.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), None);
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land.example.com/x/mod.ts")
+        .unwrap();
+    assert_eq!(auth_tokens.get(&fixture), None);
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land:8080/x/mod.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), None);
+  }
+
+  #[test]
+  fn test_auth_tokens_multiple() {
+    let auth_tokens =
+      AuthTokens::new(Some("abc123@deno.land;def456@example.com".to_string()));
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land/x/mod.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), Some("abc123".to_string()));
+    let fixture =
+      ModuleSpecifier::resolve_url("http://example.com/a/file.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), Some("def456".to_string()));
+  }
+
+  #[test]
+  fn test_auth_tokens_port() {
+    let auth_tokens =
+      AuthTokens::new(Some("abc123@deno.land:8080".to_string()));
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land/x/mod.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), None);
+    let fixture =
+      ModuleSpecifier::resolve_url("http://deno.land:8080/x/mod.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), Some("abc123".to_string()));
+  }
 }
diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs
@@ -208,6 +208,41 @@ mod integration {
     assert_eq!("noColor false", util::strip_ansi_codes(stdout_str));
   }
 
+  #[test]
+  fn auth_tokens() {
+    let _g = util::http_server();
+    let output = util::deno_cmd()
+      .current_dir(util::root_path())
+      .arg("run")
+      .arg("http://127.0.0.1:4551/cli/tests/001_hello.js")
+      .stdout(std::process::Stdio::piped())
+      .stderr(std::process::Stdio::piped())
+      .spawn()
+      .unwrap()
+      .wait_with_output()
+      .unwrap();
+    assert!(!output.status.success());
+    let stdout_str = std::str::from_utf8(&output.stdout).unwrap().trim();
+    assert!(stdout_str.is_empty());
+    let stderr_str = std::str::from_utf8(&output.stderr).unwrap().trim();
+    assert!(stderr_str.contains("Import 'http://127.0.0.1:4551/cli/tests/001_hello.js' failed: 404 Not Found"));
+
+    let output = util::deno_cmd()
+      .current_dir(util::root_path())
+      .arg("run")
+      .arg("http://127.0.0.1:4551/cli/tests/001_hello.js")
+      .env("DENO_AUTH_TOKENS", "abcdef123456789@127.0.0.1:4551")
+      .stdout(std::process::Stdio::piped())
+      .stderr(std::process::Stdio::piped())
+      .spawn()
+      .unwrap()
+      .wait_with_output()
+      .unwrap();
+    assert!(output.status.success());
+    let stdout_str = std::str::from_utf8(&output.stdout).unwrap().trim();
+    assert_eq!(util::strip_ansi_codes(stdout_str), "Hello World");
+  }
+
   #[cfg(unix)]
   #[test]
   pub fn test_raw_tty() {

diff --git a/docs/images/private-github-new-token.png b/docs/images/private-github-new-token.png
diff --git a/docs/images/private-github-token-display.png b/docs/images/private-github-token-display.png
diff --git a/docs/images/private-pat.png b/docs/images/private-pat.png