From 6c632abc24daafab7d4cbe979073fd71e27406d1 Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Thu, 9 Jun 2022 15:11:10 +0800 Subject: [PATCH 1/9] chore(build): bump toolchain to 2022-06-09 Signed-off-by: Alex Chi --- ci/scripts/unit-test.sh | 2 +- rust-toolchain | 2 +- src/common/src/lib.rs | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/scripts/unit-test.sh b/ci/scripts/unit-test.sh index a0a33b63dbfb6..931f334e66956 100755 --- a/ci/scripts/unit-test.sh +++ b/ci/scripts/unit-test.sh @@ -13,7 +13,7 @@ cargo doc --document-private-items --no-deps echo "--- Run unit tests with coverage" # use tee to disable progress bar -cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints -- --no-fail-fast | tee +cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints -- --no-fail-fast 2> >(tee) echo "--- Run doctest" cargo test --doc diff --git a/rust-toolchain b/rust-toolchain index bab7dd315e346..7e041bcf7fafa 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2022-05-24 +nightly-2022-06-09 diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs index dd3c7bd0f1ea0..54c793e700f54 100644 --- a/src/common/src/lib.rs +++ b/src/common/src/lib.rs @@ -36,6 +36,7 @@ #![feature(test)] #![feature(trusted_len)] #![feature(allocator_api)] +#![feature(nll)] #[macro_use] pub mod error; From 528398c0f962195ab1291984eba1d6cee33a3ea2 Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Thu, 9 Jun 2022 15:22:35 +0800 Subject: [PATCH 2/9] fix clippy Signed-off-by: Alex Chi --- src/common/src/lib.rs | 1 - src/connector/src/kinesis/source/reader.rs | 6 ++--- .../vector_op/agg/general_sorted_grouper.rs | 2 +- src/expr/src/vector_op/ascii.rs | 2 +- src/frontend/src/binder/expr/mod.rs | 2 +- src/frontend/src/expr/function_call.rs | 2 +- .../src/scheduler/distributed/stage.rs | 2 +- src/frontend/src/scheduler/local.rs | 2 +- src/frontend/test_runner/src/lib.rs | 4 ++-- src/meta/src/barrier/command.rs | 2 +- src/meta/src/barrier/recovery.rs | 4 ++-- src/prost/helpers/src/generate.rs | 4 ++-- src/sqlparser/src/parser.rs | 2 +- src/storage/src/hummock/compactor_tests.rs | 6 ++--- .../src/hummock/iterator/forward_user.rs | 4 ++-- src/storage/src/table/cell_based_table.rs | 22 +++---------------- .../managed_state/aggregation/string_agg.rs | 6 ++--- 17 files changed, 27 insertions(+), 46 deletions(-) diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs index 54c793e700f54..dd3c7bd0f1ea0 100644 --- a/src/common/src/lib.rs +++ b/src/common/src/lib.rs @@ -36,7 +36,6 @@ #![feature(test)] #![feature(trusted_len)] #![feature(allocator_api)] -#![feature(nll)] #[macro_use] pub mod error; diff --git a/src/connector/src/kinesis/source/reader.rs b/src/connector/src/kinesis/source/reader.rs index 81f485cdefcd7..7394ac5bb49db 100644 --- a/src/connector/src/kinesis/source/reader.rs +++ b/src/connector/src/kinesis/source/reader.rs @@ -150,13 +150,11 @@ impl KinesisSplitReader { async fn get_records( &mut self, ) -> core::result::Result> { - let resp = self - .client + self.client .get_records() .set_shard_iterator(self.shard_iter.take()) .send() - .await; - resp + .await } } diff --git a/src/expr/src/vector_op/agg/general_sorted_grouper.rs b/src/expr/src/vector_op/agg/general_sorted_grouper.rs index e54a624bc502d..9f5bb75933769 100644 --- a/src/expr/src/vector_op/agg/general_sorted_grouper.rs +++ b/src/expr/src/vector_op/agg/general_sorted_grouper.rs @@ -111,7 +111,7 @@ impl EqGroups { use std::collections::BinaryHeap; let mut heap = BinaryHeap::new(); for (ci, column) in columns.iter().enumerate() { - if let Some(ri) = column.starting_indices().get(0) { + if let Some(ri) = column.starting_indices().first() { heap.push(Reverse((ri, ci, 0))); } } diff --git a/src/expr/src/vector_op/ascii.rs b/src/expr/src/vector_op/ascii.rs index 0156325399b5a..2fbb629144e0f 100644 --- a/src/expr/src/vector_op/ascii.rs +++ b/src/expr/src/vector_op/ascii.rs @@ -16,7 +16,7 @@ use crate::Result; #[inline(always)] pub fn ascii(s: &str) -> Result { - Ok(s.as_bytes().get(0).map(|x| *x as i32).unwrap_or(0)) + Ok(s.as_bytes().first().map(|x| *x as i32).unwrap_or(0)) } #[cfg(test)] diff --git a/src/frontend/src/binder/expr/mod.rs b/src/frontend/src/binder/expr/mod.rs index 49d49b17f9e10..fb5fcfbbcf262 100644 --- a/src/frontend/src/binder/expr/mod.rs +++ b/src/frontend/src/binder/expr/mod.rs @@ -204,7 +204,7 @@ impl Binder { if return_type.is_numeric() { return Ok(expr); } - return Err(ErrorCode::InvalidInputSyntax(format!("+ {:?}", return_type)).into()); + Err(ErrorCode::InvalidInputSyntax(format!("+ {:?}", return_type)).into()) } pub(super) fn bind_trim( diff --git a/src/frontend/src/expr/function_call.rs b/src/frontend/src/expr/function_call.rs index 871227b4f55a7..09823d6733d7d 100644 --- a/src/frontend/src/expr/function_call.rs +++ b/src/frontend/src/expr/function_call.rs @@ -57,7 +57,7 @@ impl std::fmt::Debug for FunctionCall { ExprType::Cast => { assert_eq!(self.inputs.len(), 1); self.inputs[0].fmt(f)?; - return write!(f, "::{:?}", self.return_type); + write!(f, "::{:?}", self.return_type) } ExprType::Add => debug_binary_op(f, "+", &self.inputs), ExprType::Subtract => debug_binary_op(f, "-", &self.inputs), diff --git a/src/frontend/src/scheduler/distributed/stage.rs b/src/frontend/src/scheduler/distributed/stage.rs index 38b4118c75da2..73230e178b68d 100644 --- a/src/frontend/src/scheduler/distributed/stage.rs +++ b/src/frontend/src/scheduler/distributed/stage.rs @@ -393,7 +393,7 @@ impl StageRunner { let children = execution_plan_node .children .iter() - .map(|e| self.convert_plan_node(&*e, task_id)) + .map(|e| self.convert_plan_node(e, task_id)) .collect(); PlanNodeProst { diff --git a/src/frontend/src/scheduler/local.rs b/src/frontend/src/scheduler/local.rs index ca988895759fb..122876a941014 100644 --- a/src/frontend/src/scheduler/local.rs +++ b/src/frontend/src/scheduler/local.rs @@ -114,7 +114,7 @@ impl LocalQueryExecution { let children = execution_plan_node .children .iter() - .map(|e| self.convert_plan_node(&*e)) + .map(|e| self.convert_plan_node(e)) .collect::>>()?; Ok(PlanNodeProst { diff --git a/src/frontend/test_runner/src/lib.rs b/src/frontend/test_runner/src/lib.rs index 3265467cda448..8e4955a90ec7f 100644 --- a/src/frontend/test_runner/src/lib.rs +++ b/src/frontend/test_runner/src/lib.rs @@ -485,12 +485,12 @@ fn check_err(ctx: &str, expected_err: &Option, actual_err: &Option { let mut dependent_table_actors = Vec::with_capacity(table_sink_map.len()); for (table_id, actors) in table_sink_map { diff --git a/src/meta/src/barrier/recovery.rs b/src/meta/src/barrier/recovery.rs index 423148d5fdce2..c44a6d62c8e18 100644 --- a/src/meta/src/barrier/recovery.rs +++ b/src/meta/src/barrier/recovery.rs @@ -116,14 +116,14 @@ where .expect("Retry until recovery success."); debug!("recovery success"); - return ( + ( new_epoch, self.fragment_manager.all_chain_actor_ids().await, responses .into_iter() .flat_map(|r| r.create_mview_progress) .collect(), - ); + ) } /// Sync all sources in compute nodes, the local source manager in compute nodes may be dirty diff --git a/src/prost/helpers/src/generate.rs b/src/prost/helpers/src/generate.rs index 1001d68631e55..8c198eb5e8a86 100644 --- a/src/prost/helpers/src/generate.rs +++ b/src/prost/helpers/src/generate.rs @@ -124,10 +124,10 @@ pub fn implement(field: &Field) -> TokenStream2 { } } - return quote! { + quote! { #[inline(always)] pub fn #getter_fn_name(&self) -> &#ty { &self.#field_name } - }; + } } diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs index 01cbcd2313c86..0bbada7b86260 100644 --- a/src/sqlparser/src/parser.rs +++ b/src/sqlparser/src/parser.rs @@ -1356,7 +1356,7 @@ impl Parser { let all = self.parse_keyword(Keyword::ALL); let distinct = self.parse_keyword(Keyword::DISTINCT); if all && distinct { - return parser_err!("Cannot specify both ALL and DISTINCT".to_string()); + parser_err!("Cannot specify both ALL and DISTINCT".to_string()) } else { Ok(distinct) } diff --git a/src/storage/src/hummock/compactor_tests.rs b/src/storage/src/hummock/compactor_tests.rs index 6f69f8a5ff3c9..b5e770b79d0b6 100644 --- a/src/storage/src/hummock/compactor_tests.rs +++ b/src/storage/src/hummock/compactor_tests.rs @@ -48,15 +48,15 @@ mod tests { ..Default::default() }); let sstable_store = mock_sstable_store(); - let storage = HummockStorage::with_default_stats( + + HummockStorage::with_default_stats( options.clone(), sstable_store, hummock_meta_client.clone(), Arc::new(StateStoreMetrics::unused()), ) .await - .unwrap(); - storage + .unwrap() } #[tokio::test] diff --git a/src/storage/src/hummock/iterator/forward_user.rs b/src/storage/src/hummock/iterator/forward_user.rs index 9322ac1d74304..a6acbdd3705aa 100644 --- a/src/storage/src/hummock/iterator/forward_user.rs +++ b/src/storage/src/hummock/iterator/forward_user.rs @@ -249,8 +249,8 @@ impl UserIterator { // Handle multi-version self.last_key.clear(); // Handle range scan when key > end_key - let res = self.next().await; - res + + self.next().await } /// Indicates whether the iterator can be used. diff --git a/src/storage/src/table/cell_based_table.rs b/src/storage/src/table/cell_based_table.rs index 29abfa9cb70fc..e5140c5e9fef1 100644 --- a/src/storage/src/table/cell_based_table.rs +++ b/src/storage/src/table/cell_based_table.rs @@ -63,6 +63,7 @@ pub struct CellBasedTable { column_ids: Vec, /// Statistics. + #[allow(dead_code)] stats: Arc, /// Indices of distribution keys in pk for computing value meta. None if value meta is not @@ -310,13 +311,7 @@ impl CellBasedTable { // The returned iterator will iterate data from a snapshot corresponding to the given `epoch` pub async fn iter(&self, epoch: u64) -> StorageResult> { - CellBasedTableRowIter::new( - &self.keyspace, - self.column_descs.clone(), - epoch, - self.stats.clone(), - ) - .await + CellBasedTableRowIter::new(&self.keyspace, self.column_descs.clone(), epoch).await } pub async fn iter_with_pk( @@ -328,7 +323,6 @@ impl CellBasedTable { self.keyspace.clone(), self.column_descs.clone(), epoch, - self.stats.clone(), pk_descs, ) .await @@ -405,7 +399,6 @@ impl CellBasedTable { self.column_descs.clone(), (start_key, end_key), epoch, - self.stats.clone(), ) .await } @@ -431,7 +424,6 @@ impl CellBasedTable { self.column_descs.clone(), (start_key, next_key), epoch, - self.stats.clone(), ) .await } @@ -461,8 +453,6 @@ pub struct CellBasedTableRowIter { iter: StripPrefixIterator, /// Cell-based row deserializer cell_based_row_deserializer: CellBasedRowDeserializer, - /// Statistics - _stats: Arc, } impl CellBasedTableRowIter { @@ -470,7 +460,6 @@ impl CellBasedTableRowIter { keyspace: &Keyspace, table_descs: Vec, epoch: u64, - _stats: Arc, ) -> StorageResult { keyspace.state_store().wait_epoch(epoch).await?; @@ -481,7 +470,6 @@ impl CellBasedTableRowIter { let iter = Self { iter, cell_based_row_deserializer, - _stats, }; Ok(iter) } @@ -491,7 +479,6 @@ impl CellBasedTableRowIter { table_descs: Vec, serialized_pk_bounds: R, epoch: u64, - _stats: Arc, ) -> StorageResult where R: RangeBounds + Send, @@ -507,7 +494,6 @@ impl CellBasedTableRowIter { let iter = Self { iter, cell_based_row_deserializer, - _stats, }; Ok(iter) } @@ -572,11 +558,9 @@ impl DedupPkCellBasedTableRowIter { keyspace: Keyspace, table_descs: Vec, epoch: u64, - _stats: Arc, pk_descs: &[OrderedColumnDesc], ) -> StorageResult { - let inner = - CellBasedTableRowIter::new(&keyspace, table_descs.clone(), epoch, _stats).await?; + let inner = CellBasedTableRowIter::new(&keyspace, table_descs.clone(), epoch).await?; let (data_types, order_types) = pk_descs .iter() diff --git a/src/stream/src/executor/managed_state/aggregation/string_agg.rs b/src/stream/src/executor/managed_state/aggregation/string_agg.rs index cd1a763dc356d..7ed4db893a9e3 100644 --- a/src/stream/src/executor/managed_state/aggregation/string_agg.rs +++ b/src/stream/src/executor/managed_state/aggregation/string_agg.rs @@ -291,7 +291,8 @@ mod tests { .map(|(ord, idx)| OrderPair::new(idx, ord)) .collect::>(); let sort_key_serializer = OrderedArraysSerializer::new(order_pairs); - let managed_state = ManagedStringAggState::new( + + ManagedStringAggState::new( keyspace.clone(), row_count, sort_key_indices, @@ -300,8 +301,7 @@ mod tests { sort_key_serializer, ) .await - .unwrap(); - managed_state + .unwrap() } #[tokio::test] From 8fd8f02cc62b1678a65daf0095cf1fb831748f46 Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Fri, 10 Jun 2022 12:41:17 +0800 Subject: [PATCH 3/9] update tag Signed-off-by: Alex Chi --- ci/build-ci-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh index 7fa7a3ab0e8b6..be3c470089537 100755 --- a/ci/build-ci-image.sh +++ b/ci/build-ci-image.sh @@ -9,7 +9,7 @@ cd "$DIR" cat ../rust-toolchain # shellcheck disable=SC2155 export RUST_TOOLCHAIN=$(cat ../rust-toolchain) -export BUILD_ENV_VERSION=v20220609 +export BUILD_ENV_VERSION=v20220610 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}" docker build -t ${BUILD_TAG} --build-arg "RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" . From 02108c46e720cde349bf6127590f1a1983f4091d Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Fri, 10 Jun 2022 12:43:19 +0800 Subject: [PATCH 4/9] update login Signed-off-by: Alex Chi --- ci/build-ci-image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh index be3c470089537..8512fd76507bd 100755 --- a/ci/build-ci-image.sh +++ b/ci/build-ci-image.sh @@ -13,4 +13,5 @@ export BUILD_ENV_VERSION=v20220610 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}" docker build -t ${BUILD_TAG} --build-arg "RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" . +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/x5u3w5h6 docker push ${BUILD_TAG} From 414f94444f23a48301e239222adcde9076536b90 Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Fri, 10 Jun 2022 12:57:37 +0800 Subject: [PATCH 5/9] shrink image size Signed-off-by: Alex Chi --- ci/Dockerfile | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ci/Dockerfile b/ci/Dockerfile index 3185a57ecb660..7072bc9131f5c 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -6,7 +6,8 @@ ARG RUST_TOOLCHAIN RUN apt-get update -yy && \ DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl \ - openssl libssl-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld -yy + openssl libssl-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld -yy \ + && rm -rf /var/lib/{apt,dpkg,cache,log}/ SHELL ["/bin/bash", "-c"] @@ -21,14 +22,11 @@ WORKDIR /risingwave ENV PATH /root/.cargo/bin/:$PATH +# add required rustup components +RUN rustup component add rustfmt llvm-tools-preview clippy + # install build tools -RUN cargo install cargo-llvm-cov cargo-nextest cargo-udeps cargo-hakari cargo-sort cargo-make +RUN cargo install cargo-llvm-cov cargo-nextest cargo-udeps cargo-hakari cargo-sort cargo-make && cargo install --git https://github.com/risinglightdb/sqllogictest-rs --features bin # pre-cache required crates RUN git clone https://github.com/singularity-data/risingwave && cd risingwave && cargo fetch && cd .. && rm -rf risingwave - -# add required rustup components -RUN rustup component add rustfmt llvm-tools-preview clippy - -# install sqllogictest -RUN cargo install --git https://github.com/risinglightdb/sqllogictest-rs --features bin From ec0ce35fdefa256d002b70d4e1052bfd9d742858 Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Fri, 10 Jun 2022 13:00:22 +0800 Subject: [PATCH 6/9] clear cache Signed-off-by: Alex Chi --- ci/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/Dockerfile b/ci/Dockerfile index 7072bc9131f5c..4d9a3a5c4277b 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -26,7 +26,9 @@ ENV PATH /root/.cargo/bin/:$PATH RUN rustup component add rustfmt llvm-tools-preview clippy # install build tools -RUN cargo install cargo-llvm-cov cargo-nextest cargo-udeps cargo-hakari cargo-sort cargo-make && cargo install --git https://github.com/risinglightdb/sqllogictest-rs --features bin +RUN cargo install cargo-llvm-cov cargo-nextest cargo-udeps cargo-hakari cargo-sort cargo-make cargo-cache \ + && cargo install --git https://github.com/risinglightdb/sqllogictest-rs --features bin \ + && cargo cache -a # pre-cache required crates -RUN git clone https://github.com/singularity-data/risingwave && cd risingwave && cargo fetch && cd .. && rm -rf risingwave +RUN git clone https://github.com/singularity-data/risingwave && cd risingwave && cargo fetch && cd .. && rm -rf risingwave && cargo cache -g From 88e66f15089a486195e2938ffe1f168c96df5b88 Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Fri, 10 Jun 2022 13:14:03 +0800 Subject: [PATCH 7/9] update tag Signed-off-by: Alex Chi --- ci/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 2cbb3adea35bf..a7fd0f6007b1c 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.9" services: rw-build-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v202206091256 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20220610 # build: # context: .. # dockerfile: ./ci/Dockerfile From 5e281ff19ec2b0701d8daf3c1f553c08e162a877 Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Fri, 10 Jun 2022 13:24:59 +0800 Subject: [PATCH 8/9] fix coverage tag Signed-off-by: Alex Chi --- src/prost/helpers/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/prost/helpers/src/lib.rs b/src/prost/helpers/src/lib.rs index 90dc099bea024..dc78e12b1d241 100644 --- a/src/prost/helpers/src/lib.rs +++ b/src/prost/helpers/src/lib.rs @@ -25,7 +25,6 @@ use proc_macro2::TokenStream as TokenStream2; use proc_macro_error::{proc_macro_error, ResultExt}; use syn::{DataStruct, DeriveInput}; -#[cfg_attr(coverage, no_coverage)] mod generate; // This attribute will be placed before any pb types, including messages and enums. From 6132bd1da16441fe47bb87bd62035cfa340a1734 Mon Sep 17 00:00:00 2001 From: Alex Chi Date: Fri, 10 Jun 2022 13:36:14 +0800 Subject: [PATCH 9/9] fix e2e test Signed-off-by: Alex Chi --- e2e_test/batch/functions/concat.slt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e_test/batch/functions/concat.slt b/e2e_test/batch/functions/concat.slt index 0df2dff91acca..02b75dcd42039 100644 --- a/e2e_test/batch/functions/concat.slt +++ b/e2e_test/batch/functions/concat.slt @@ -9,12 +9,12 @@ ab query T select concat(NULL); ---- - +(empty) query T select concat(NULL, NULL); ---- - +(empty) query T select concat(1, 1.01);