From a95a48fe9b7b0a9f074a63109329d05d84360a67 Mon Sep 17 00:00:00 2001
From: Darun Seethammagari <darun@near.org>
Date: Tue, 6 Aug 2024 10:20:46 -0700
Subject: [PATCH] fix: Receiver blocks stream would not query today (#987)

The current date used to start the receiver blocks backfill would have
hour, minute, second, and nanosecond data. This meant when it reached
today's date, it may potentially be ahead. This would lead to today not
being queried. This PR fixes this by ensuring the current date is at the
very start of the day, which guarantees it passes the <= check.
---
 block-streamer/src/block_stream.rs            |  8 ++-----
 .../receiver_blocks_processor.rs              | 22 +++++++++++++------
 .../src/server/block_streamer_service.rs      |  2 +-
 block-streamer/src/test_utils.rs              |  7 +++++-
 4 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/block-streamer/src/block_stream.rs b/block-streamer/src/block_stream.rs
index c9a25ac62..561e2fc75 100644
--- a/block-streamer/src/block_stream.rs
+++ b/block-streamer/src/block_stream.rs
@@ -533,11 +533,7 @@ mod tests {
                 predicate::eq("near-lake-data-mainnet".to_string()),
                 predicate::eq("000091940840/block.json"),
             )
-            .returning(move |_, _| {
-                Ok(crate::test_utils::generate_block_with_timestamp(
-                    "2023-12-09",
-                ))
-            });
+            .returning(move |_, _| Ok(crate::test_utils::generate_block_with_date("2023-12-09")));
 
         let mut mock_graphql_client = crate::graphql::client::GraphQLClient::default();
 
@@ -663,7 +659,7 @@ mod tests {
                 predicate::eq("000107503704/block.json"),
             )
             .returning(move |_, _| {
-                Ok(crate::test_utils::generate_block_with_timestamp(
+                Ok(crate::test_utils::generate_block_with_date(
                     &chrono::Utc::now().format("%Y-%m-%d").to_string(),
                 ))
             });
diff --git a/block-streamer/src/receiver_blocks/receiver_blocks_processor.rs b/block-streamer/src/receiver_blocks/receiver_blocks_processor.rs
index 29dfb55ec..4076f8ee9 100644
--- a/block-streamer/src/receiver_blocks/receiver_blocks_processor.rs
+++ b/block-streamer/src/receiver_blocks/receiver_blocks_processor.rs
@@ -1,6 +1,6 @@
 use anyhow::Context;
 use async_stream::try_stream;
-use chrono::{DateTime, Duration, TimeZone, Utc};
+use chrono::{DateTime, Duration, TimeZone, Timelike, Utc};
 use near_lake_framework::near_indexer_primitives;
 use regex::Regex;
 
@@ -165,7 +165,15 @@ impl ReceiverBlocksProcessor {
         try_stream! {
             let start_date = self.get_nearest_block_date(start_block_height).await?;
             let contract_pattern_type = ContractPatternType::from(contract_pattern.as_str());
-            let mut current_date = start_date;
+            let mut current_date = start_date
+                .with_hour(0)
+                .unwrap()
+                .with_minute(0)
+                .unwrap()
+                .with_second(0)
+                .unwrap()
+                .with_nanosecond(0)
+                .unwrap();
 
             while current_date <= Utc::now() {
                 let base_64_bitmaps: Vec<Base64Bitmap> = self.query_base_64_bitmaps(&contract_pattern_type, &current_date).await?;
@@ -288,13 +296,13 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn collect_block_heights_from_one_day() {
+    async fn collect_block_heights_from_today() {
         let mut mock_s3_client = crate::s3_client::S3Client::default();
         mock_s3_client
             .expect_get_text_file()
             .returning(move |_, _| {
                 Ok(crate::test_utils::generate_block_with_timestamp(
-                    &Utc::now().format("%Y-%m-%d").to_string(),
+                    &Utc::now().format("%Y-%m-%dT%H:%M:%S").to_string(),
                 ))
             });
 
@@ -326,7 +334,7 @@ mod tests {
             .expect_get_text_file()
             .returning(move |_, _| {
                 Ok(crate::test_utils::generate_block_with_timestamp(
-                    &Utc::now().format("%Y-%m-%d").to_string(),
+                    &Utc::now().format("%Y-%m-%dT%H:%M:%S").to_string(),
                 ))
             });
 
@@ -362,8 +370,8 @@ mod tests {
             .expect_get_text_file()
             .returning(move |_, _| {
                 Ok(crate::test_utils::generate_block_with_timestamp(
-                    &(Utc::now() - Duration::days(2))
-                        .format("%Y-%m-%d")
+                    &(Utc::now() - Duration::days(2) + Duration::minutes(10))
+                        .format("%Y-%m-%dT%H:%M:%S")
                         .to_string(),
                 ))
             });
diff --git a/block-streamer/src/server/block_streamer_service.rs b/block-streamer/src/server/block_streamer_service.rs
index 4dd3f01ae..986d3b0b4 100644
--- a/block-streamer/src/server/block_streamer_service.rs
+++ b/block-streamer/src/server/block_streamer_service.rs
@@ -285,7 +285,7 @@ mod tests {
                 predicate::always(),
             )
             .returning(move |_, _| {
-                Ok(crate::test_utils::generate_block_with_timestamp(
+                Ok(crate::test_utils::generate_block_with_date(
                     &chrono::Utc::now().format("%Y-%m-%d").to_string(),
                 ))
             });
diff --git a/block-streamer/src/test_utils.rs b/block-streamer/src/test_utils.rs
index 0039cd616..daec93230 100644
--- a/block-streamer/src/test_utils.rs
+++ b/block-streamer/src/test_utils.rs
@@ -184,11 +184,16 @@ pub fn utc_date_time_from_date_string(date: &str) -> chrono::DateTime<chrono::Ut
     chrono::TimeZone::from_utc_datetime(&chrono::Utc, &naive_date_time)
 }
 
-pub fn generate_block_with_timestamp(date: &str) -> String {
+pub fn generate_block_with_date(date: &str) -> String {
     let naive_date = chrono::NaiveDate::parse_from_str(date, "%Y-%m-%d")
         .unwrap()
         .and_hms_opt(0, 0, 0)
         .unwrap();
+    return generate_block_with_timestamp(&naive_date.format("%Y-%m-%dT%H:%M:%S").to_string());
+}
+
+pub fn generate_block_with_timestamp(date: &str) -> String {
+    let naive_date = chrono::NaiveDateTime::parse_from_str(date, "%Y-%m-%dT%H:%M:%S").unwrap();
 
     let date_time_utc = chrono::Utc.from_utc_datetime(&naive_date).timestamp() * 1_000_000_000;