Skip to content

Commit

Permalink
fix: Receiver blocks stream would not query today (#987)
Browse files Browse the repository at this point in the history
The current date used to start the receiver blocks backfill would have
hour, minute, second, and nanosecond data. This meant when it reached
today's date, it may potentially be ahead. This would lead to today not
being queried. This PR fixes this by ensuring the current date is at the
very start of the day, which guarantees it passes the <= check.
  • Loading branch information
darunrs authored Aug 6, 2024
1 parent cdc5066 commit a95a48f
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 15 deletions.
8 changes: 2 additions & 6 deletions block-streamer/src/block_stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -533,11 +533,7 @@ mod tests {
predicate::eq("near-lake-data-mainnet".to_string()),
predicate::eq("000091940840/block.json"),
)
.returning(move |_, _| {
Ok(crate::test_utils::generate_block_with_timestamp(
"2023-12-09",
))
});
.returning(move |_, _| Ok(crate::test_utils::generate_block_with_date("2023-12-09")));

let mut mock_graphql_client = crate::graphql::client::GraphQLClient::default();

Expand Down Expand Up @@ -663,7 +659,7 @@ mod tests {
predicate::eq("000107503704/block.json"),
)
.returning(move |_, _| {
Ok(crate::test_utils::generate_block_with_timestamp(
Ok(crate::test_utils::generate_block_with_date(
&chrono::Utc::now().format("%Y-%m-%d").to_string(),
))
});
Expand Down
22 changes: 15 additions & 7 deletions block-streamer/src/receiver_blocks/receiver_blocks_processor.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use anyhow::Context;
use async_stream::try_stream;
use chrono::{DateTime, Duration, TimeZone, Utc};
use chrono::{DateTime, Duration, TimeZone, Timelike, Utc};
use near_lake_framework::near_indexer_primitives;
use regex::Regex;

Expand Down Expand Up @@ -165,7 +165,15 @@ impl ReceiverBlocksProcessor {
try_stream! {
let start_date = self.get_nearest_block_date(start_block_height).await?;
let contract_pattern_type = ContractPatternType::from(contract_pattern.as_str());
let mut current_date = start_date;
let mut current_date = start_date
.with_hour(0)
.unwrap()
.with_minute(0)
.unwrap()
.with_second(0)
.unwrap()
.with_nanosecond(0)
.unwrap();

while current_date <= Utc::now() {
let base_64_bitmaps: Vec<Base64Bitmap> = self.query_base_64_bitmaps(&contract_pattern_type, &current_date).await?;
Expand Down Expand Up @@ -288,13 +296,13 @@ mod tests {
}

#[tokio::test]
async fn collect_block_heights_from_one_day() {
async fn collect_block_heights_from_today() {
let mut mock_s3_client = crate::s3_client::S3Client::default();
mock_s3_client
.expect_get_text_file()
.returning(move |_, _| {
Ok(crate::test_utils::generate_block_with_timestamp(
&Utc::now().format("%Y-%m-%d").to_string(),
&Utc::now().format("%Y-%m-%dT%H:%M:%S").to_string(),
))
});

Expand Down Expand Up @@ -326,7 +334,7 @@ mod tests {
.expect_get_text_file()
.returning(move |_, _| {
Ok(crate::test_utils::generate_block_with_timestamp(
&Utc::now().format("%Y-%m-%d").to_string(),
&Utc::now().format("%Y-%m-%dT%H:%M:%S").to_string(),
))
});

Expand Down Expand Up @@ -362,8 +370,8 @@ mod tests {
.expect_get_text_file()
.returning(move |_, _| {
Ok(crate::test_utils::generate_block_with_timestamp(
&(Utc::now() - Duration::days(2))
.format("%Y-%m-%d")
&(Utc::now() - Duration::days(2) + Duration::minutes(10))
.format("%Y-%m-%dT%H:%M:%S")
.to_string(),
))
});
Expand Down
2 changes: 1 addition & 1 deletion block-streamer/src/server/block_streamer_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ mod tests {
predicate::always(),
)
.returning(move |_, _| {
Ok(crate::test_utils::generate_block_with_timestamp(
Ok(crate::test_utils::generate_block_with_date(
&chrono::Utc::now().format("%Y-%m-%d").to_string(),
))
});
Expand Down
7 changes: 6 additions & 1 deletion block-streamer/src/test_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -184,11 +184,16 @@ pub fn utc_date_time_from_date_string(date: &str) -> chrono::DateTime<chrono::Ut
chrono::TimeZone::from_utc_datetime(&chrono::Utc, &naive_date_time)
}

pub fn generate_block_with_timestamp(date: &str) -> String {
pub fn generate_block_with_date(date: &str) -> String {
let naive_date = chrono::NaiveDate::parse_from_str(date, "%Y-%m-%d")
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap();
return generate_block_with_timestamp(&naive_date.format("%Y-%m-%dT%H:%M:%S").to_string());
}

pub fn generate_block_with_timestamp(date: &str) -> String {
let naive_date = chrono::NaiveDateTime::parse_from_str(date, "%Y-%m-%dT%H:%M:%S").unwrap();

let date_time_utc = chrono::Utc.from_utc_datetime(&naive_date).timestamp() * 1_000_000_000;

Expand Down

0 comments on commit a95a48f

Please sign in to comment.