pass migrations to statements + add optimized queries

blockscout · Sep 6, 2024 · 9f967ff · 9f967ff
1 parent e2c1a4a
commit 9f967ff
Show file tree

Hide file tree

Showing 24 changed files with 721 additions and 408 deletions.
diff --git a/stats/stats/src/charts/counters/average_block_time.rs b/stats/stats/src/charts/counters/average_block_time.rs
@@ -1,8 +1,11 @@
 use crate::{
-    data_source::kinds::{
-        data_manipulation::map::MapToString,
-        local_db::DirectPointLocalDbChartSource,
-        remote_db::{PullOne, RemoteDatabaseSource, StatementForOne},
+    data_source::{
+        kinds::{
+            data_manipulation::map::MapToString,
+            local_db::DirectPointLocalDbChartSource,
+            remote_db::{PullOne, RemoteDatabaseSource, StatementForOne},
+        },
+        types::BlockscoutMigrations,
     },
     ChartProperties, MissingDatePolicy, Named,
 };
@@ -14,7 +17,7 @@ use sea_orm::{DbBackend, Statement};
 pub struct AverageBlockTimeStatement;
 
 impl StatementForOne for AverageBlockTimeStatement {
-    fn get_statement() -> Statement {
+    fn get_statement(_: &BlockscoutMigrations) -> Statement {
         Statement::from_sql_and_values(
             DbBackend::Postgres,
             r#"

diff --git a/stats/stats/src/charts/counters/completed_txns.rs b/stats/stats/src/charts/counters/completed_txns.rs
@@ -1,7 +1,10 @@
 use crate::{
-    data_source::kinds::{
-        local_db::DirectPointLocalDbChartSource,
-        remote_db::{PullOne, RemoteDatabaseSource, StatementForOne},
+    data_source::{
+        kinds::{
+            local_db::DirectPointLocalDbChartSource,
+            remote_db::{PullOne, RemoteDatabaseSource, StatementForOne},
+        },
+        types::BlockscoutMigrations,
     },
     ChartProperties, MissingDatePolicy, Named,
 };
@@ -13,31 +16,57 @@ use sea_orm::{DbBackend, Statement};
 pub struct CompletedTxnsStatement;
 
 impl StatementForOne for CompletedTxnsStatement {
-    fn get_statement() -> Statement {
-        Statement::from_string(
-            DbBackend::Postgres,
-            r#"
-                SELECT
-                    (all_success - all_success_dropped)::TEXT AS value,
-                    last_block_date AS date 
-                FROM (
-                    SELECT (
-                        SELECT COUNT(*) AS all_success
-                        FROM transactions t
-                        WHERE t.status = 1
-                    ), (
-                        SELECT COUNT(*) as all_success_dropped
-                        FROM transactions t
-                        JOIN blocks b ON t.block_hash = b.hash
-                        WHERE t.status = 1 AND b.consensus = false
-                    ), (
-                        SELECT MAX(b.timestamp)::DATE AS last_block_date
-                        FROM blocks b
-                        WHERE b.consensus = true
-                    )
-                ) AS sub
-            "#,
-        )
+    fn get_statement(completed_migrations: &BlockscoutMigrations) -> Statement {
+        if completed_migrations.denormalization {
+            Statement::from_string(
+                DbBackend::Postgres,
+                r#"
+                    SELECT
+                        (all_success - all_success_dropped)::TEXT AS value,
+                        last_block_date AS date 
+                    FROM (
+                        SELECT (
+                            SELECT COUNT(*) AS all_success
+                            FROM transactions t
+                            WHERE t.status = 1
+                        ), (
+                            SELECT COUNT(*) as all_success_dropped
+                            FROM transactions t
+                            WHERE t.status = 1 AND t.block_consensus = false
+                        ), (
+                            SELECT MAX(b.timestamp)::DATE AS last_block_date
+                            FROM blocks b
+                            WHERE b.consensus = true
+                        )
+                    ) AS sub
+                "#,
+            )
+        } else {
+            Statement::from_string(
+                DbBackend::Postgres,
+                r#"
+                    SELECT
+                        (all_success - all_success_dropped)::TEXT AS value,
+                        last_block_date AS date 
+                    FROM (
+                        SELECT (
+                            SELECT COUNT(*) AS all_success
+                            FROM transactions t
+                            WHERE t.status = 1
+                        ), (
+                            SELECT COUNT(*) as all_success_dropped
+                            FROM transactions t
+                            JOIN blocks b ON t.block_hash = b.hash
+                            WHERE t.status = 1 AND b.consensus = false
+                        ), (
+                            SELECT MAX(b.timestamp)::DATE AS last_block_date
+                            FROM blocks b
+                            WHERE b.consensus = true
+                        )
+                    ) AS sub
+                "#,
+            )
+        }
     }
 }
 
@@ -68,11 +97,16 @@ pub type CompletedTxns = DirectPointLocalDbChartSource<CompletedTxnsRemote, Prop
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::tests::simple_test::simple_test_counter;
+    use crate::tests::simple_test::simple_test_counter_with_migration_variants;
 
     #[tokio::test]
     #[ignore = "needs database to run"]
     async fn update_completed_txns() {
-        simple_test_counter::<CompletedTxns>("update_completed_txns", "46", None).await;
+        simple_test_counter_with_migration_variants::<CompletedTxns>(
+            "update_completed_txns",
+            "46",
+            None,
+        )
+        .await;
     }
 }
diff --git a/stats/stats/src/charts/counters/total_addresses.rs b/stats/stats/src/charts/counters/total_addresses.rs
@@ -1,7 +1,10 @@
 use crate::{
-    data_source::kinds::{
-        local_db::DirectPointLocalDbChartSource,
-        remote_db::{PullOne, RemoteDatabaseSource, StatementForOne},
+    data_source::{
+        kinds::{
+            local_db::DirectPointLocalDbChartSource,
+            remote_db::{PullOne, RemoteDatabaseSource, StatementForOne},
+        },
+        types::BlockscoutMigrations,
     },
     ChartProperties, MissingDatePolicy, Named,
 };
@@ -12,7 +15,7 @@ use sea_orm::{DbBackend, Statement};
 pub struct TotalAddressesStatement;
 
 impl StatementForOne for TotalAddressesStatement {
-    fn get_statement() -> Statement {
+    fn get_statement(_: &BlockscoutMigrations) -> Statement {
         Statement::from_string(
             DbBackend::Postgres,
             r#"

diff --git a/stats/stats/src/charts/counters/total_tokens.rs b/stats/stats/src/charts/counters/total_tokens.rs
@@ -1,7 +1,10 @@
 use crate::{
-    data_source::kinds::{
-        local_db::DirectPointLocalDbChartSource,
-        remote_db::{PullOne, RemoteDatabaseSource, StatementForOne},
+    data_source::{
+        kinds::{
+            local_db::DirectPointLocalDbChartSource,
+            remote_db::{PullOne, RemoteDatabaseSource, StatementForOne},
+        },
+        types::BlockscoutMigrations,
     },
     ChartProperties, MissingDatePolicy, Named,
 };
@@ -13,7 +16,7 @@ use sea_orm::{DbBackend, Statement};
 pub struct TotalTokensStatement;
 
 impl StatementForOne for TotalTokensStatement {
-    fn get_statement() -> Statement {
+    fn get_statement(_: &BlockscoutMigrations) -> Statement {
         Statement::from_string(
             DbBackend::Postgres,
             r#"

diff --git a/stats/stats/src/charts/lines/active_accounts.rs b/stats/stats/src/charts/lines/active_accounts.rs
@@ -3,11 +3,14 @@
 use std::ops::Range;
 
 use crate::{
-    data_source::kinds::{
-        local_db::{
-            parameters::update::batching::parameters::Batch30Days, DirectVecLocalDbChartSource,
+    data_source::{
+        kinds::{
+            local_db::{
+                parameters::update::batching::parameters::Batch30Days, DirectVecLocalDbChartSource,
+            },
+            remote_db::{PullAllWithAndSort, RemoteDatabaseSource, StatementFromRange},
         },
-        remote_db::{PullAllWithAndSort, RemoteDatabaseSource, StatementFromRange},
+        types::BlockscoutMigrations,
     },
     utils::sql_with_range_filter_opt,
     ChartProperties, Named,
@@ -20,24 +23,46 @@ use sea_orm::{prelude::*, DbBackend, Statement};
 pub struct ActiveAccountsStatement;
 
 impl StatementFromRange for ActiveAccountsStatement {
-    fn get_statement(range: Option<Range<DateTimeUtc>>) -> Statement {
-        sql_with_range_filter_opt!(
-            DbBackend::Postgres,
-            r#"
-                SELECT
-                    DATE(blocks.timestamp) as date,
-                    COUNT(DISTINCT from_address_hash)::TEXT as value
-                FROM transactions
-                JOIN blocks on transactions.block_hash = blocks.hash
-                WHERE
-                    blocks.timestamp != to_timestamp(0) AND
-                    blocks.consensus = true {filter}
-                GROUP BY date(blocks.timestamp);
-            "#,
-            [],
-            "blocks.timestamp",
-            range
-        )
+    fn get_statement(
+        range: Option<Range<DateTimeUtc>>,
+        completed_migrations: &BlockscoutMigrations,
+    ) -> Statement {
+        if completed_migrations.denormalization {
+            sql_with_range_filter_opt!(
+                DbBackend::Postgres,
+                r#"
+                    SELECT
+                        DATE(block_timestamp) as date,
+                        COUNT(DISTINCT from_address_hash)::TEXT as value
+                    FROM transactions
+                    WHERE
+                        block_timestamp != to_timestamp(0) AND
+                        block_consensus = true {filter}
+                    GROUP BY date(block_timestamp);
+                "#,
+                [],
+                "block_timestamp",
+                range
+            )
+        } else {
+            sql_with_range_filter_opt!(
+                DbBackend::Postgres,
+                r#"
+                    SELECT
+                        DATE(blocks.timestamp) as date,
+                        COUNT(DISTINCT from_address_hash)::TEXT as value
+                    FROM transactions
+                    JOIN blocks on transactions.block_hash = blocks.hash
+                    WHERE
+                        blocks.timestamp != to_timestamp(0) AND
+                        blocks.consensus = true {filter}
+                    GROUP BY date(blocks.timestamp);
+                "#,
+                [],
+                "blocks.timestamp",
+                range
+            )
+        }
     }
 }
 
@@ -65,14 +90,14 @@ pub type ActiveAccounts =
 
 #[cfg(test)]
 mod tests {
-    use crate::tests::simple_test::simple_test_chart;
+    use crate::tests::simple_test::simple_test_chart_with_migration_variants;
 
     use super::ActiveAccounts;
 
     #[tokio::test]
     #[ignore = "needs database to run"]
     async fn update_active_accounts() {
-        simple_test_chart::<ActiveAccounts>(
+        simple_test_chart_with_migration_variants::<ActiveAccounts>(
             "update_active_accounts",
             vec![
                 ("2022-11-09", "1"),

diff --git a/stats/stats/src/charts/lines/average_block_rewards.rs b/stats/stats/src/charts/lines/average_block_rewards.rs
@@ -1,18 +1,21 @@
 use std::ops::Range;
 
 use crate::{
-    data_source::kinds::{
-        data_manipulation::{
-            map::{MapParseTo, MapToString},
-            resolutions::average::AverageLowerResolution,
-        },
-        local_db::{
-            parameters::update::batching::parameters::{
-                Batch30Days, Batch30Weeks, Batch30Years, Batch36Months,
+    data_source::{
+        kinds::{
+            data_manipulation::{
+                map::{MapParseTo, MapToString},
+                resolutions::average::AverageLowerResolution,
+            },
+            local_db::{
+                parameters::update::batching::parameters::{
+                    Batch30Days, Batch30Weeks, Batch30Years, Batch36Months,
+                },
+                DirectVecLocalDbChartSource,
             },
-            DirectVecLocalDbChartSource,
+            remote_db::{PullAllWithAndSort, RemoteDatabaseSource, StatementFromRange},
         },
-        remote_db::{PullAllWithAndSort, RemoteDatabaseSource, StatementFromRange},
+        types::BlockscoutMigrations,
     },
     define_and_impl_resolution_properties,
     types::timespans::{Month, Week, Year},
@@ -31,7 +34,7 @@ const ETH: i64 = 1_000_000_000_000_000_000;
 pub struct AverageBlockRewardsQuery;
 
 impl StatementFromRange for AverageBlockRewardsQuery {
-    fn get_statement(range: Option<Range<DateTimeUtc>>) -> Statement {
+    fn get_statement(range: Option<Range<DateTimeUtc>>, _: &BlockscoutMigrations) -> Statement {
         sql_with_range_filter_opt!(
             DbBackend::Postgres,
             r#"

diff --git a/stats/stats/src/charts/lines/average_block_size.rs b/stats/stats/src/charts/lines/average_block_size.rs
@@ -1,18 +1,21 @@
 use std::ops::Range;
 
 use crate::{
-    data_source::kinds::{
-        data_manipulation::{
-            map::{MapParseTo, MapToString},
-            resolutions::average::AverageLowerResolution,
-        },
-        local_db::{
-            parameters::update::batching::parameters::{
-                Batch30Days, Batch30Weeks, Batch30Years, Batch36Months,
+    data_source::{
+        kinds::{
+            data_manipulation::{
+                map::{MapParseTo, MapToString},
+                resolutions::average::AverageLowerResolution,
+            },
+            local_db::{
+                parameters::update::batching::parameters::{
+                    Batch30Days, Batch30Weeks, Batch30Years, Batch36Months,
+                },
+                DirectVecLocalDbChartSource,
             },
-            DirectVecLocalDbChartSource,
+            remote_db::{PullAllWithAndSort, RemoteDatabaseSource, StatementFromRange},
         },
-        remote_db::{PullAllWithAndSort, RemoteDatabaseSource, StatementFromRange},
+        types::BlockscoutMigrations,
     },
     define_and_impl_resolution_properties,
     types::timespans::{Month, Week, Year},
@@ -29,7 +32,7 @@ use super::new_blocks::{NewBlocksInt, NewBlocksMonthlyInt};
 pub struct AverageBlockSizeStatement;
 
 impl StatementFromRange for AverageBlockSizeStatement {
-    fn get_statement(range: Option<Range<DateTimeUtc>>) -> Statement {
+    fn get_statement(range: Option<Range<DateTimeUtc>>, _: &BlockscoutMigrations) -> Statement {
         sql_with_range_filter_opt!(
             DbBackend::Postgres,
             r#"