feat(fetcher): add github app vault token

.gitignore

@@ -2,4 +2,7 @@
 Cargo.lock
 etherface.log
 .DS_Store
-/signature-dump
+/signature-dump
+
+/.idea
+/.vscode

deploy/prod/etherface/deploy.yaml

@@ -6,8 +6,7 @@ metadata:
     app: etherface
     component: etherface
 spec:
-  replicas: 0
-
+  replicas: 1
   selector:
     matchLabels:
       app: etherface
@@ -27,6 +26,9 @@ spec:
           envFrom:
             - secretRef:
                 name: etherface
+          env:
+            - name: VAULT_ADDR
+              value: "http://vault-ui.guardian.svc.cluster.local:8200"
           resources:
             requests:
               cpu: "100m"

deploy/prod/etherface/secret.yaml

@@ -7,6 +7,6 @@ metadata:
 type: Opaque
 stringData:
   ETHERFACE_TOKEN_ETHERSCAN: "<ETHERFACE_TOKEN_ETHERSCAN>"
-  ETHERFACE_TOKENS_GITHUB: "<ETHERFACE_TOKENS_GITHUB>"
+  ETHERFACE_TOKENS_GITHUB: ""
   ETHERFACE_DATABASE_URL: "postgres://<PG_USER>:<PG_PASS>@<PG_HOST>:5432/<PG_DB>"
   ETHERFACE_REST_ADDRESS: "https://api.etherface.io"

etherface-lib/Cargo.toml

@@ -1,28 +1,34 @@
 [package]
 name = "etherface-lib"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-reqwest = { version = "0.11", features = ["blocking", "json"] }
+reqwest = { version = "^0.11", features = ["blocking", "json"] }
 serde = { version = "1.0", features = ["derive"] }
 chrono = { version = "0.4", features = ["serde"] }
 serde_json = "1.0"
 thiserror = "1.0"
 log = "0.4"
-toml = "0.5"
+toml = "0.8"
 url = "2.0"
-hyperx = "1.0"
-select = "0.5"
+hyperx = "1.4.0"
+select = "0.6"
 sha3 = "0.10"
 lazy_static = "1.0"
-regex = "1.0"
+regex = "^1.0"
 dotenv = "0.15"
 
 semver = "1.0"
 lenient_semver = "0.4"
 
 diesel = { version = "1.4", features = ["postgres", "chrono", "r2d2"] }
 diesel-derive-enum = { version = "1.1.2", features = ["postgres"] }
+
+futures = "^0.3"
+rustify = "0.5.3"
+rustify_derive = "0.5.2"
+derive_builder = "0.12.0"
+vaultrs = "^0.6"

etherface-lib/src/api/github/handler/search.rs

@@ -4,8 +4,7 @@ use crate::api::github::page::Page;
 use crate::api::github::GithubClient;
 use crate::error::Error;
 use crate::model::GithubRepository;
-use chrono::Date;
-use chrono::Utc;
+use chrono::NaiveDate;
 
 pub struct SearchHandler<'a> {
     ghc: &'a GithubClient,
@@ -23,19 +22,20 @@ impl<'a> SearchHandler<'a> {
     }
 
     /// Returns the deserialized JSON `/search/repositories?q=language:solidity created:{date}` response.
-    pub fn solidity_repos_created_at(&self, date: Date<Utc>) -> Result<Vec<GithubRepository>, Error> {
+    pub fn solidity_repos_created_at(&self, date: NaiveDate) -> Result<Vec<GithubRepository>, Error> {
         self.repos(&format!("language:solidity created:{}", date.format("%Y-%m-%d")))
     }
 
     /// Returns the deserialized JSON `/search/repositories?q=language:solidity pushed:{date}` response.
-    pub fn solidity_repos_updated_at(&self, date: Date<Utc>) -> Result<Vec<GithubRepository>, Error> {
+    pub fn solidity_repos_updated_at(&self, date: NaiveDate) -> Result<Vec<GithubRepository>, Error> {
         self.repos(&format!("language:solidity pushed:{}", date.format("%Y-%m-%d")))
     }
 }
 
 #[cfg(test)]
 mod tests {
     use crate::api::github::GithubClient;
+    use chrono::NaiveDate;
     use chrono::TimeZone;
     use chrono::Utc;
 
@@ -56,7 +56,8 @@ mod tests {
         let ghc = GithubClient::new().unwrap();
 
         // https://api.github.com/search/repositories?q=language:solidity%20created:2022-01-01&per_page=100
-        let search = ghc.search().solidity_repos_created_at(Utc.ymd(2022, 1, 1)).unwrap();
+        let search =
+            ghc.search().solidity_repos_created_at(NaiveDate::from_ymd_opt(2022, 1, 1).unwrap()).unwrap();
         assert_eq!(search.len(), 96);
     }
 
@@ -65,7 +66,8 @@ mod tests {
         let ghc = GithubClient::new().unwrap();
 
         // https://api.github.com/search/repositories?q=language:solidity%20pushed:2022-01-01&per_page=100
-        let search = ghc.search().solidity_repos_updated_at(Utc.ymd(2022, 1, 1)).unwrap();
+        let search =
+            ghc.search().solidity_repos_updated_at(NaiveDate::from_ymd_opt(2022, 1, 1).unwrap()).unwrap();
         assert_eq!(search.len(), 81);
     }
 }

etherface-lib/src/api/github/token.rs

@@ -1,14 +1,15 @@
 //! GitHub API token manager.
-//! 
-//! Because GitHub has a ratelimit of 5000 requests / hour, which for crawling purposes is very little, 
+//!
+//! Because GitHub has a ratelimit of 5000 requests / hour, which for crawling purposes is very little,
 //! Etherface uses multiple GitHub API tokens. For that some logic reagarding which token should be actively
 //! used is needed, which this module does. In basic terms all tokens are read from the config file and stored
 //! in an internal token pool. Initially the first token in the token pool will be used for all GitHub API
-//! requests. If, however, the active token is drained, i.e. all 5000 requests / hour have been reached, the 
+//! requests. If, however, the active token is drained, i.e. all 5000 requests / hour have been reached, the
 //! token manager will automatically find a new token in the pool to temporarily replace the old active token
 //! (see the [`refresh`] function). As such the GitHub API client doesn't have to worry about token managment.
 
 use crate::api::github::GITHUB_RATELIMIT_URL;
+use crate::api::vault::VaultManager;
 use crate::api::RequestHandler;
 use crate::api::TokenManagerResponseHandler;
 use crate::config::Config;
@@ -40,6 +41,7 @@ pub(crate) struct TokenManager {
     pub active: String,
     pool: Vec<String>,
     request_handler: Box<RequestHandler>,
+    vault_manager: VaultManager,
 }
 
 impl TokenManager {
@@ -51,6 +53,13 @@ impl TokenManager {
             active: tokens[0].clone(),
             pool: tokens,
             request_handler: Box::new(RequestHandler::new()),
+            vault_manager: match VaultManager::new() {
+                Ok(vault_manager) => vault_manager,
+                Err(e) => {
+                    warn!("Failed to initialize VaultManager: {}", e);
+                    return Err(Error::VaultConfigError);
+                }
+            },
         };
         manager.cleanup()?; // Make sure we have only valid tokens before returning the TokenManager
 
@@ -82,6 +91,17 @@ impl TokenManager {
             }
         }
 
+        let dynamic_token = match self.vault_manager.get_token() {
+            Ok(token) => token,
+            Err(e) => {
+                warn!("Failed to get token from vault: {}", e);
+                return Err(Error::VaultConfigError);
+            }
+        };
+        if let Ok(ratelimit) = self.execute(&dynamic_token) {
+            valid_tokens.push((&dynamic_token, ratelimit.core.remaining));
+        }
+
         if valid_tokens.is_empty() {
             return Err(Error::GithubTokenPoolEmpty);
         }
@@ -138,6 +158,8 @@ impl TokenManager {
             .execute_deser_token::<TokenManagerResponseHandler, RatelimitRoot>(GITHUB_RATELIMIT_URL, token)?
             .resources)
     }
+
+    // request github device code
 }
 
 #[cfg(test)]

etherface-lib/src/api/mod.rs

@@ -14,6 +14,7 @@ use std::cell::RefCell;
 pub mod etherscan;
 pub mod fourbyte;
 pub mod github;
+pub mod vault;
 
 struct RequestHandler {
     client: Client,

etherface-lib/src/api/vault/mod.rs

@@ -0,0 +1,130 @@
+use crate::config::Config;
+use derive_builder::Builder;
+use futures::executor::block_on;
+use rustify_derive::Endpoint;
+use serde::Deserialize;
+use std::{collections::HashMap, error::Error, fs::read_to_string, result::Result};
+use vaultrs::{
+    api,
+    auth::kubernetes::login,
+    client::{Client, VaultClient, VaultClientSettingsBuilder},
+};
+
+const SERVICE_ACCOUNT_TOKEN_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token";
+
+#[derive(Debug, Builder, Endpoint)]
+#[endpoint(
+    path = "{self.mount}/{self.path}",
+    method = "GET",
+    response = "GithubResponse",
+    builder = "true"
+)]
+#[builder(setter(into))]
+struct GithubRequest {
+    #[endpoint(skip)]
+    mount: String,
+    path: String,
+    #[endpoint(query)]
+    org_name: String,
+}
+
+#[derive(Deserialize, Debug)]
+struct GithubResponse {
+    pub request_id: String,
+    pub lease_id: String,
+    pub renewable: bool,
+    pub lease_duration: i32,
+    pub data: GithubResponseData,
+    pub wrap_info: Option<String>,
+    pub warnings: Option<String>,
+    pub auth: Option<String>,
+}
+
+#[derive(Deserialize, Debug)]
+struct GithubResponseData {
+    pub expires_at: String,
+    pub installation_id: i32,
+    pub org_name: String,
+    pub permissions: HashMap<String, String>,
+    pub repository_selection: String,
+    pub token: String,
+}
+
+pub(crate) struct VaultManager {
+    client: VaultClient,
+    token: Option<GithubResponse>,
+    mount: String,
+    path: String,
+    org_name: String,
+}
+
+impl VaultManager {
+    /// Returns a new token manager.
+    pub fn new() -> Result<Self, Box<dyn Error>> {
+        let vault_config = Config::new()?.vault;
+
+        let mut client =
+            VaultClient::new(VaultClientSettingsBuilder::default().address(vault_config.address).build()?)?;
+
+        match vault_config.auth.method.as_str() {
+            "kubernetes" => {
+                let jwt = read_to_string(SERVICE_ACCOUNT_TOKEN_PATH)?;
+                let auth = block_on(login(&client, &vault_config.auth.path, &vault_config.auth.role, &jwt))?;
+                client.set_token(&auth.client_token);
+            }
+            "token" => match vault_config.auth.token {
+                Some(token) => {
+                    client.set_token(token.as_str());
+                }
+                None => {
+                    return Err("Token auth method requires a token".into());
+                }
+            },
+            _ => {
+                return Err(format!("Unsupported auth method: {}", vault_config.auth.method).into());
+            }
+        }
+
+        let mut manager = VaultManager {
+            client,
+            mount: vault_config.secret.mount,
+            path: vault_config.secret.path,
+            org_name: vault_config.secret.org_name,
+            token: None,
+        };
+
+        Ok(manager)
+    }
+
+    pub fn get_token(&mut self) -> Result<String, Box<dyn Error>> {
+        // if token is expired or not exist, renew it
+        if let Some(token) = &self.token {
+            if token.data.expires_at < chrono::Utc::now().to_rfc3339() {
+                block_on(self.renew_token())?
+            }
+        } else {
+            block_on(self.renew_token())?
+        }
+
+        return match &self.token {
+            Some(token) => Ok(token.data.token.clone()),
+            None => Err("Token not found".into()),
+        };
+    }
+
+    pub async fn renew_token(&mut self) -> Result<(), Box<dyn Error>> {
+        let endpoint = GithubRequestBuilder::default()
+            .mount(self.mount.clone())
+            .path(self.path.clone())
+            .org_name(self.org_name.clone())
+            .build()?;
+
+        match api::exec_with_no_result(&self.client, endpoint).await {
+            Ok(response) => {
+                self.token = Some(response);
+                Ok(())
+            }
+            Err(e) => Err(Box::new(e)),
+        }
+    }
+}