Skip to content

Commit

Permalink
[data ingestion] decrease batch size for analytics (#18804)
Browse files Browse the repository at this point in the history
  • Loading branch information
phoenix-o authored Jul 25, 2024
1 parent 69a9e6b commit a4812b9
Showing 1 changed file with 13 additions and 3 deletions.
16 changes: 13 additions & 3 deletions crates/sui-analytics-indexer/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use sui_analytics_indexer::{
analytics_metrics::AnalyticsMetrics, errors::AnalyticsIndexerError, make_analytics_processor,
AnalyticsIndexerConfig,
};
use sui_data_ingestion_core::setup_single_workflow;
use sui_data_ingestion_core::{setup_single_workflow, ReaderOptions};
use tokio::signal;
use tracing::info;

Expand Down Expand Up @@ -37,8 +37,18 @@ async fn main() -> Result<()> {
.map_err(|e| AnalyticsIndexerError::GenericError(e.to_string()))?;
let watermark = processor.last_committed_checkpoint().unwrap_or_default() + 1;

let (executor, exit_sender) =
setup_single_workflow(processor, remote_store_url, watermark, 1, None).await?;
let reader_options = ReaderOptions {
batch_size: 10,
..Default::default()
};
let (executor, exit_sender) = setup_single_workflow(
processor,
remote_store_url,
watermark,
1,
Some(reader_options),
)
.await?;

tokio::spawn(async {
signal::ctrl_c()
Expand Down

0 comments on commit a4812b9

Please sign in to comment.