diff --git a/.gitignore b/.gitignore index 220a301..ad51bcc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,7 @@ Cargo.lock etherface.log .DS_Store -/signature-dump \ No newline at end of file +/signature-dump + +/.idea +/.vscode \ No newline at end of file diff --git a/.idea/etherface.iml b/.idea/etherface.iml new file mode 100644 index 0000000..6b02f8f --- /dev/null +++ b/.idea/etherface.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..f88e300 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/deploy/prod/etherface/deploy.yaml b/deploy/prod/etherface/deploy.yaml index 2734c26..3be1eb8 100644 --- a/deploy/prod/etherface/deploy.yaml +++ b/deploy/prod/etherface/deploy.yaml @@ -6,8 +6,7 @@ metadata: app: etherface component: etherface spec: - replicas: 0 - + replicas: 1 selector: matchLabels: app: etherface @@ -27,6 +26,9 @@ spec: envFrom: - secretRef: name: etherface + env: + - name: VAULT_ADDR + value: "http://vault-ui.guardian.svc.cluster.local:8200" resources: requests: cpu: "100m" diff --git a/deploy/prod/etherface/secret.yaml b/deploy/prod/etherface/secret.yaml index 580b58e..8dccd16 100644 --- a/deploy/prod/etherface/secret.yaml +++ b/deploy/prod/etherface/secret.yaml @@ -7,6 +7,6 @@ metadata: type: Opaque stringData: ETHERFACE_TOKEN_ETHERSCAN: "" - ETHERFACE_TOKENS_GITHUB: "" + ETHERFACE_TOKENS_GITHUB: "" ETHERFACE_DATABASE_URL: "postgres://:@:5432/" ETHERFACE_REST_ADDRESS: "https://api.etherface.io" diff --git a/etherface-lib/Cargo.toml b/etherface-lib/Cargo.toml index a66393e..7bfe76d 100644 --- a/etherface-lib/Cargo.toml +++ b/etherface-lib/Cargo.toml @@ -1,24 +1,24 @@ [package] name = "etherface-lib" -version = "0.1.0" +version = "0.2.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -reqwest = { version = "0.11", features = ["blocking", "json"] } +reqwest = { version = "^0.11", features = ["blocking", "json"] } serde = { version = "1.0", features = ["derive"] } chrono = { version = "0.4", features = ["serde"] } serde_json = "1.0" thiserror = "1.0" log = "0.4" -toml = "0.5" +toml = "0.8" url = "2.0" -hyperx = "1.0" -select = "0.5" +hyperx = "1.4.0" +select = "0.6" sha3 = "0.10" lazy_static = "1.0" -regex = "1.0" +regex = "^1.0" dotenv = "0.15" semver = "1.0" @@ -26,3 +26,9 @@ lenient_semver = "0.4" diesel = { version = "1.4", features = ["postgres", "chrono", "r2d2"] } diesel-derive-enum = { version = "1.1.2", features = ["postgres"] } + +futures = "^0.3" +rustify = "0.5.3" +rustify_derive = "0.5.2" +derive_builder = "0.12.0" +vaultrs = "^0.6" diff --git a/etherface-lib/src/api/github/handler/search.rs b/etherface-lib/src/api/github/handler/search.rs index e7a8d0f..7964380 100644 --- a/etherface-lib/src/api/github/handler/search.rs +++ b/etherface-lib/src/api/github/handler/search.rs @@ -4,8 +4,7 @@ use crate::api::github::page::Page; use crate::api::github::GithubClient; use crate::error::Error; use crate::model::GithubRepository; -use chrono::Date; -use chrono::Utc; +use chrono::NaiveDate; pub struct SearchHandler<'a> { ghc: &'a GithubClient, @@ -23,12 +22,12 @@ impl<'a> SearchHandler<'a> { } /// Returns the deserialized JSON `/search/repositories?q=language:solidity created:{date}` response. - pub fn solidity_repos_created_at(&self, date: Date) -> Result, Error> { + pub fn solidity_repos_created_at(&self, date: NaiveDate) -> Result, Error> { self.repos(&format!("language:solidity created:{}", date.format("%Y-%m-%d"))) } /// Returns the deserialized JSON `/search/repositories?q=language:solidity pushed:{date}` response. - pub fn solidity_repos_updated_at(&self, date: Date) -> Result, Error> { + pub fn solidity_repos_updated_at(&self, date: NaiveDate) -> Result, Error> { self.repos(&format!("language:solidity pushed:{}", date.format("%Y-%m-%d"))) } } @@ -36,6 +35,7 @@ impl<'a> SearchHandler<'a> { #[cfg(test)] mod tests { use crate::api::github::GithubClient; + use chrono::NaiveDate; use chrono::TimeZone; use chrono::Utc; @@ -56,7 +56,8 @@ mod tests { let ghc = GithubClient::new().unwrap(); // https://api.github.com/search/repositories?q=language:solidity%20created:2022-01-01&per_page=100 - let search = ghc.search().solidity_repos_created_at(Utc.ymd(2022, 1, 1)).unwrap(); + let search = + ghc.search().solidity_repos_created_at(NaiveDate::from_ymd_opt(2022, 1, 1).unwrap()).unwrap(); assert_eq!(search.len(), 96); } @@ -65,7 +66,8 @@ mod tests { let ghc = GithubClient::new().unwrap(); // https://api.github.com/search/repositories?q=language:solidity%20pushed:2022-01-01&per_page=100 - let search = ghc.search().solidity_repos_updated_at(Utc.ymd(2022, 1, 1)).unwrap(); + let search = + ghc.search().solidity_repos_updated_at(NaiveDate::from_ymd_opt(2022, 1, 1).unwrap()).unwrap(); assert_eq!(search.len(), 81); } } diff --git a/etherface-lib/src/api/github/token.rs b/etherface-lib/src/api/github/token.rs index ff4e40f..8b034f4 100644 --- a/etherface-lib/src/api/github/token.rs +++ b/etherface-lib/src/api/github/token.rs @@ -1,14 +1,15 @@ //! GitHub API token manager. -//! -//! Because GitHub has a ratelimit of 5000 requests / hour, which for crawling purposes is very little, +//! +//! Because GitHub has a ratelimit of 5000 requests / hour, which for crawling purposes is very little, //! Etherface uses multiple GitHub API tokens. For that some logic reagarding which token should be actively //! used is needed, which this module does. In basic terms all tokens are read from the config file and stored //! in an internal token pool. Initially the first token in the token pool will be used for all GitHub API -//! requests. If, however, the active token is drained, i.e. all 5000 requests / hour have been reached, the +//! requests. If, however, the active token is drained, i.e. all 5000 requests / hour have been reached, the //! token manager will automatically find a new token in the pool to temporarily replace the old active token //! (see the [`refresh`] function). As such the GitHub API client doesn't have to worry about token managment. use crate::api::github::GITHUB_RATELIMIT_URL; +use crate::api::vault::VaultManager; use crate::api::RequestHandler; use crate::api::TokenManagerResponseHandler; use crate::config::Config; @@ -40,6 +41,7 @@ pub(crate) struct TokenManager { pub active: String, pool: Vec, request_handler: Box, + vault_manager: VaultManager, } impl TokenManager { @@ -51,6 +53,13 @@ impl TokenManager { active: tokens[0].clone(), pool: tokens, request_handler: Box::new(RequestHandler::new()), + vault_manager: match VaultManager::new() { + Ok(vault_manager) => vault_manager, + Err(e) => { + warn!("Failed to initialize VaultManager: {}", e); + return Err(Error::VaultConfigError); + } + }, }; manager.cleanup()?; // Make sure we have only valid tokens before returning the TokenManager @@ -82,6 +91,17 @@ impl TokenManager { } } + let dynamic_token = match self.vault_manager.get_token() { + Ok(token) => token, + Err(e) => { + warn!("Failed to get token from vault: {}", e); + return Err(Error::VaultConfigError); + } + }; + if let Ok(ratelimit) = self.execute(&dynamic_token) { + valid_tokens.push((&dynamic_token, ratelimit.core.remaining)); + } + if valid_tokens.is_empty() { return Err(Error::GithubTokenPoolEmpty); } @@ -138,6 +158,8 @@ impl TokenManager { .execute_deser_token::(GITHUB_RATELIMIT_URL, token)? .resources) } + + // request github device code } #[cfg(test)] diff --git a/etherface-lib/src/api/mod.rs b/etherface-lib/src/api/mod.rs index b664e17..ed930bd 100644 --- a/etherface-lib/src/api/mod.rs +++ b/etherface-lib/src/api/mod.rs @@ -14,6 +14,7 @@ use std::cell::RefCell; pub mod etherscan; pub mod fourbyte; pub mod github; +pub mod vault; struct RequestHandler { client: Client, diff --git a/etherface-lib/src/api/vault/mod.rs b/etherface-lib/src/api/vault/mod.rs new file mode 100644 index 0000000..63b05e3 --- /dev/null +++ b/etherface-lib/src/api/vault/mod.rs @@ -0,0 +1,130 @@ +use crate::config::Config; +use derive_builder::Builder; +use futures::executor::block_on; +use rustify_derive::Endpoint; +use serde::Deserialize; +use std::{collections::HashMap, error::Error, fs::read_to_string, result::Result}; +use vaultrs::{ + api, + auth::kubernetes::login, + client::{Client, VaultClient, VaultClientSettingsBuilder}, +}; + +const SERVICE_ACCOUNT_TOKEN_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token"; + +#[derive(Debug, Builder, Endpoint)] +#[endpoint( + path = "{self.mount}/{self.path}", + method = "GET", + response = "GithubResponse", + builder = "true" +)] +#[builder(setter(into))] +struct GithubRequest { + #[endpoint(skip)] + mount: String, + path: String, + #[endpoint(query)] + org_name: String, +} + +#[derive(Deserialize, Debug)] +struct GithubResponse { + pub request_id: String, + pub lease_id: String, + pub renewable: bool, + pub lease_duration: i32, + pub data: GithubResponseData, + pub wrap_info: Option, + pub warnings: Option, + pub auth: Option, +} + +#[derive(Deserialize, Debug)] +struct GithubResponseData { + pub expires_at: String, + pub installation_id: i32, + pub org_name: String, + pub permissions: HashMap, + pub repository_selection: String, + pub token: String, +} + +pub(crate) struct VaultManager { + client: VaultClient, + token: Option, + mount: String, + path: String, + org_name: String, +} + +impl VaultManager { + /// Returns a new token manager. + pub fn new() -> Result> { + let vault_config = Config::new()?.vault; + + let mut client = + VaultClient::new(VaultClientSettingsBuilder::default().address(vault_config.address).build()?)?; + + match vault_config.auth.method.as_str() { + "kubernetes" => { + let jwt = read_to_string(SERVICE_ACCOUNT_TOKEN_PATH)?; + let auth = block_on(login(&client, &vault_config.auth.path, &vault_config.auth.role, &jwt))?; + client.set_token(&auth.client_token); + } + "token" => match vault_config.auth.token { + Some(token) => { + client.set_token(token.as_str()); + } + None => { + return Err("Token auth method requires a token".into()); + } + }, + _ => { + return Err(format!("Unsupported auth method: {}", vault_config.auth.method).into()); + } + } + + let mut manager = VaultManager { + client, + mount: vault_config.secret.mount, + path: vault_config.secret.path, + org_name: vault_config.secret.org_name, + token: None, + }; + + Ok(manager) + } + + pub fn get_token(&mut self) -> Result> { + // if token is expired or not exist, renew it + if let Some(token) = &self.token { + if token.data.expires_at < chrono::Utc::now().to_rfc3339() { + block_on(self.renew_token())? + } + } else { + block_on(self.renew_token())? + } + + return match &self.token { + Some(token) => Ok(token.data.token.clone()), + None => Err("Token not found".into()), + }; + } + + pub async fn renew_token(&mut self) -> Result<(), Box> { + let endpoint = GithubRequestBuilder::default() + .mount(self.mount.clone()) + .path(self.path.clone()) + .org_name(self.org_name.clone()) + .build()?; + + match api::exec_with_no_result(&self.client, endpoint).await { + Ok(response) => { + self.token = Some(response); + Ok(()) + } + Err(e) => Err(Box::new(e)), + } + } +} diff --git a/etherface-lib/src/config.rs b/etherface-lib/src/config.rs index cfaebd3..f17ebb2 100644 --- a/etherface-lib/src/config.rs +++ b/etherface-lib/src/config.rs @@ -3,8 +3,6 @@ //! Reads all content from `.env` into [`Config`] for all sub-modules to use. use crate::error::Error; -use dotenv::dotenv; -use std::path::Path; pub struct Config { /// Database URL with the following structure `postgres://username:password@host/database_name`. @@ -18,6 +16,44 @@ pub struct Config { /// Etherface REST API address, e.g. pub rest_address: String, + + pub vault: VaultConfig, +} + +pub struct VaultConfig { + /// Vault address. + pub address: String, + + /// Vault auth + pub auth: VaultAuth, + + /// Vault secret + pub secret: VaultSecret, +} + +pub struct VaultAuth { + /// Vault auth method + pub method: String, + + /// Vault auth path + pub path: String, + + /// Vault auth role + pub role: String, + + /// Vault token (optional) + pub token: Option, +} + +pub struct VaultSecret { + /// Vault mount + pub mount: String, + + /// Vault path + pub path: String, + + /// Vault param org_name + pub org_name: String, } const ENV_VAR_DATABASE_URL: &str = "ETHERFACE_DATABASE_URL"; @@ -25,6 +61,15 @@ const ENV_VAR_TOKEN_ETHERSCAN: &str = "ETHERFACE_TOKEN_ETHERSCAN"; const ENV_VAR_TOKENS_GITHUB: &str = "ETHERFACE_TOKENS_GITHUB"; const ENV_VAR_REST_ADDRESS: &str = "ETHERFACE_REST_ADDRESS"; +const ENV_VAR_VAULT_ADDR: &str = "VAULT_ADDR"; +const ENV_VAR_VAULT_AUTH_METHOD: &str = "VAULT_AUTH_METHOD"; +const ENV_VAR_VAULT_AUTH_PATH: &str = "VAULT_AUTH_PATH"; +const ENV_VAR_VAULT_AUTH_ROLE: &str = "VAULT_AUTH_ROLE"; +const ENV_VAR_VAULT_AUTH_TOKEN: &str = "VAULT_TOKEN"; +const ENV_VAR_VAULT_SECRET_MOUNT: &str = "VAULT_SECRET_MOUNT"; +const ENV_VAR_VAULT_SECRET_PATH: &str = "VAULT_SECRET_PATH"; +const ENV_VAR_VAULT_SECRET_ORG_NAME: &str = "VAULT_SECRET_ORG_NAME"; + #[inline] fn read_and_return_env_var(env_var: &'static str) -> Result { let res = std::env::var(env_var) @@ -54,15 +99,33 @@ impl Config { .map(str::to_string) .collect::>(); - if tokens_github.is_empty() { - return Err(Error::ConfigReadEmptyEnvironmentVariable(ENV_VAR_TOKENS_GITHUB)); - } + // if tokens_github.is_empty() { + // return Err(Error::ConfigReadEmptyEnvironmentVariable(ENV_VAR_TOKENS_GITHUB)); + // } + + let vault = VaultConfig { + address: read_and_return_env_var(ENV_VAR_VAULT_ADDR)?, + auth: VaultAuth { + method: read_and_return_env_var(ENV_VAR_VAULT_AUTH_METHOD) + .unwrap_or("kubernetes".to_string()), + path: read_and_return_env_var(ENV_VAR_VAULT_AUTH_PATH).unwrap_or("kubernetes".to_string()), + role: read_and_return_env_var(ENV_VAR_VAULT_AUTH_ROLE).unwrap_or("etherface".to_string()), + token: None, + }, + secret: VaultSecret { + mount: read_and_return_env_var(ENV_VAR_VAULT_SECRET_MOUNT).unwrap_or("etherface".to_string()), + path: read_and_return_env_var(ENV_VAR_VAULT_SECRET_PATH).unwrap_or("token".to_string()), + org_name: read_and_return_env_var(ENV_VAR_VAULT_SECRET_ORG_NAME) + .unwrap_or("RSS3-Network".to_string()), + }, + }; Ok(Config { database_url, tokens_github, token_etherscan, rest_address, + vault, }) } } diff --git a/etherface-lib/src/database/handler/github_repository.rs b/etherface-lib/src/database/handler/github_repository.rs index fce130d..a4580cc 100644 --- a/etherface-lib/src/database/handler/github_repository.rs +++ b/etherface-lib/src/database/handler/github_repository.rs @@ -127,7 +127,7 @@ impl<'a> GithubRepositoryHandler<'a> { github_repository .filter( updated_at - .gt(Utc::now() - chrono::Duration::days(days)) + .gt(Utc::now() - chrono::Duration::try_days(days).unwrap()) .and(solidity_ratio.gt(0.0).or(language.eq("Solidity"))), ) .get_results(self.connection) diff --git a/etherface-lib/src/database/handler/github_user.rs b/etherface-lib/src/database/handler/github_user.rs index 8fcfc9b..001708d 100644 --- a/etherface-lib/src/database/handler/github_user.rs +++ b/etherface-lib/src/database/handler/github_user.rs @@ -68,16 +68,14 @@ impl<'a> GithubUserHandler<'a> { pub fn get_solidity_repository_owners_active_in_last_n_days(&self, days: i64) -> Vec { use crate::database::schema::github_repository; - + use chrono::Duration; github_user .inner_join(github_repository::table) .filter( (github_repository::solidity_ratio.gt(0.0).or(github_repository::language.eq("Solidity"))) - .and( - github_repository::is_deleted - .eq(false) - .and(github_repository::updated_at.gt(Utc::now() - chrono::Duration::days(days))), - ), + .and(github_repository::is_deleted.eq(false).and( + github_repository::updated_at.gt(Utc::now() - Duration::try_days(days).unwrap()), + )), ) .select(github_user::all_columns) .distinct() diff --git a/etherface-lib/src/error.rs b/etherface-lib/src/error.rs index 3a75dc1..e6c2cfb 100644 --- a/etherface-lib/src/error.rs +++ b/etherface-lib/src/error.rs @@ -53,4 +53,7 @@ pub enum Error { #[error("Aborting crawling process, one or more background events disconnected from channel")] CrawlerChannelDisconnected, + + #[error("Vault config error")] + VaultConfigError, } diff --git a/etherface-rest/src/main.rs b/etherface-rest/src/main.rs index 294df3f..1e36eda 100644 --- a/etherface-rest/src/main.rs +++ b/etherface-rest/src/main.rs @@ -24,6 +24,7 @@ async fn main() -> std::io::Result<()> { .service(v1::sources_github) .service(v1::sources_etherscan) .service(v1::statistics) + .service(v1::healthcheck) .wrap(Cors::permissive()) .wrap(Logger::new("(%Ts, %s) %a: %r").log_target("v1::logger")), ) diff --git a/etherface-rest/src/v1.rs b/etherface-rest/src/v1.rs index 5958316..d99c474 100644 --- a/etherface-rest/src/v1.rs +++ b/etherface-rest/src/v1.rs @@ -1,13 +1,12 @@ -use actix_web::get; -use actix_web::web; -use actix_web::HttpResponse; -use actix_web::Responder; +use actix_web::{get, web, HttpResponse, Responder}; use etherface_lib::database::handler::DatabaseClientPooled; -use etherface_lib::model::views::ViewSignatureCountStatistics; -use etherface_lib::model::views::ViewSignatureInsertRate; -use etherface_lib::model::views::ViewSignatureKindDistribution; -use etherface_lib::model::views::ViewSignaturesPopularOnGithub; -use etherface_lib::model::SignatureKind; +use etherface_lib::model::{ + views::{ + ViewSignatureCountStatistics, ViewSignatureInsertRate, ViewSignatureKindDistribution, + ViewSignaturesPopularOnGithub, + }, + SignatureKind, +}; use serde::Deserialize; use serde::Serialize; @@ -141,4 +140,9 @@ async fn statistics(state: web::Data) -> impl Responder { }) .unwrap(), ) -} \ No newline at end of file +} + +#[get("/healthcheck")] +async fn healthcheck() -> impl Responder { + HttpResponse::Ok() +} diff --git a/etherface/src/fetcher/github.rs b/etherface/src/fetcher/github.rs index 485c2e4..184cf47 100644 --- a/etherface/src/fetcher/github.rs +++ b/etherface/src/fetcher/github.rs @@ -11,9 +11,8 @@ //! //! -use chrono::Date; use chrono::DateTime; -use chrono::TimeZone; +use chrono::NaiveDate; use chrono::Utc; use etherface_lib::api::github::GithubClient; use etherface_lib::database::handler::DatabaseClient; @@ -81,15 +80,26 @@ impl GithubCrawler { // Check if this is the first ever run and if so fetch all Solidity repositories created between 2015 // and today's date. if self.dbc.github_repository().get_total_count() == 0 { - for repo in self.search_solidity_repositories_starting_from(Utc.ymd(2015, 1, 1), true)? { + for repo in self.search_solidity_repositories_starting_from( + NaiveDate::from_ymd_opt(2015, 1, 1).unwrap(), + true, + )? { self.insert_repository_if_not_exists(&repo, false)?; } } let (tx, rx): (Sender, Receiver) = mpsc::channel(); - start_background_event(tx.clone(), Event::SearchRepositories, chrono::Duration::days(1))?; - start_background_event(tx.clone(), Event::CheckRepositories, chrono::Duration::days(21))?; - start_background_event(tx, Event::CheckUsers, chrono::Duration::days(21))?; + start_background_event( + tx.clone(), + Event::SearchRepositories, + chrono::Duration::try_days(1).unwrap(), + )?; + start_background_event( + tx.clone(), + Event::CheckRepositories, + chrono::Duration::try_days(21).unwrap(), + )?; + start_background_event(tx, Event::CheckUsers, chrono::Duration::try_days(21).unwrap())?; // Sleep a few seconds to give the background event schedulers some time to fetch data from the // database and issue events if possible @@ -100,7 +110,8 @@ impl GithubCrawler { Ok(msg) => match msg.event { Event::SearchRepositories => { debug!("Starting SearchRepositories event"); - let prev_event_date = self.dbc.github_crawler_metadata().get().last_repository_search.date(); + let prev_event_date = + self.dbc.github_crawler_metadata().get().last_repository_search.date_naive(); debug!("Prev event date: {prev_event_date}"); self.insert_recently_created_solidity_repositories(prev_event_date)?; @@ -108,8 +119,13 @@ impl GithubCrawler { // Only set if previous function calls were successful debug!("Prev event date: {}", msg.new_event_date); - self.dbc.github_crawler_metadata().update_last_repository_search_date(msg.new_event_date); - debug!("{}", self.dbc.github_crawler_metadata().get().last_repository_search.date()); + self.dbc + .github_crawler_metadata() + .update_last_repository_search_date(msg.new_event_date); + debug!( + "{}", + self.dbc.github_crawler_metadata().get().last_repository_search.date_naive() + ); } Event::CheckRepositories => { @@ -117,7 +133,9 @@ impl GithubCrawler { self.find_repository_updates(180)?; // Only set if previous function calls were successful - self.dbc.github_crawler_metadata().update_last_repository_check_date(msg.new_event_date); + self.dbc + .github_crawler_metadata() + .update_last_repository_check_date(msg.new_event_date); } Event::CheckUsers => { @@ -251,7 +269,7 @@ impl GithubCrawler { // spend further API calls to check what their languages / Solidity ratio is. // For references, from 2015 to 2018 around ~500 repos were created, whereas in 2018 alone ~3000 were // created as such we're fine if we lose a few repositories but instead improve crawling speed. - if entity.created_at.date() <= Utc.ymd(2018, 1, 1) { + if entity.created_at.naive_utc().date() <= NaiveDate::from_ymd_opt(2018, 1, 1).unwrap() { return Ok(()); } @@ -280,25 +298,25 @@ impl GithubCrawler { fn search_solidity_repositories_starting_from( &self, - mut from: Date, + mut from: NaiveDate, query_by_created: bool, ) -> Result, Error> { let mut repositories = Vec::new(); - let to = Utc::now().date(); + let to = Utc::now().date_naive(); while from <= to { match query_by_created { true => repositories.append(&mut self.ghc.search().solidity_repos_created_at(from)?), false => repositories.append(&mut self.ghc.search().solidity_repos_updated_at(from)?), } - from = from + chrono::Duration::days(1); + from = from + chrono::Duration::try_days(1).unwrap(); } Ok(repositories) } - fn insert_recently_created_solidity_repositories(&self, date: Date) -> Result<(), Error> { + fn insert_recently_created_solidity_repositories(&self, date: NaiveDate) -> Result<(), Error> { let repos = self.search_solidity_repositories_starting_from(date, true)?; debug!("Inserting {} repositories", repos.len()); @@ -309,7 +327,7 @@ impl GithubCrawler { Ok(()) } - fn upsert_recently_updated_solidity_repositories(&self, date: Date) -> Result<(), Error> { + fn upsert_recently_updated_solidity_repositories(&self, date: NaiveDate) -> Result<(), Error> { let repos = self.search_solidity_repositories_starting_from(date, false)?; debug!("Upserting {} repos", repos.len());