refactor(storage): move hummock tests to a new crate to break crate c…

…yclic dependency (#3795)

* refactor(storage): move hummock tests to a new crate to break crate cyclic dependency

* remove meta dependency for failpoint test

* Update src/storage/hummock_test/src/vacuum_tests.rs

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* add workspace-hack to the new craet

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Cargo.toml

@@ -24,6 +24,7 @@ members = [
   "src/storage",
   "src/storage/compactor",
   "src/storage/hummock_sdk",
+  "src/storage/hummock_test",
   "src/stream",
   "src/test_runner",
   "src/tests/regress",

src/storage/Cargo.toml

@@ -77,13 +77,13 @@ zstd = "0.11.2"
 
 [dev-dependencies]
 criterion = "0.3"
-risingwave_meta = { path = "../meta", features = ["test"] }
 risingwave_test_runner = { path = "../test_runner" }
 uuid = { version = "1", features = ["v4"] }
 
 [features]
 # rocksdb-local = ["rocksdb"]
 # tikv = ["tikv-client"]
+test = []
 failpoints = ["fail/failpoints"]
 
 [[bench]]

src/storage/hummock_test/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "risingwave_hummock_test"
+version = "0.1.10"
+edition = "2021"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bytes = { version = "1" }
+fail = "0.5"
+futures = { version = "0.3" }
+itertools = "0.10"
+rand = "0.8"
+risingwave_common = { path = "../../common" }
+risingwave_hummock_sdk = { path = "../hummock_sdk" }
+risingwave_meta = { path = "../../meta", features = ["test"] }
+risingwave_pb = { path = "../../prost" }
+risingwave_rpc_client = { path = "../../rpc_client" }
+risingwave_storage = { path = "..", features = ["test"] }
+tokio = { version = "=0.2.0-alpha.3", package = "madsim-tokio" }
+workspace-hack = { version = "0.1", path = "../../workspace-hack" }
+
+[features]
+failpoints = ["risingwave_storage/failpoints"]

src/storage/src/hummock/compactor_tests.rs → src/storage/hummock_test/src/compactor_tests.rs

@@ -36,15 +36,16 @@ mod tests {
     use risingwave_meta::hummock::MockHummockMetaClient;
     use risingwave_pb::hummock::{HummockVersion, TableOption};
     use risingwave_rpc_client::HummockMetaClient;
-
-    use crate::hummock::compaction_group_client::DummyCompactionGroupClient;
-    use crate::hummock::compactor::{get_remote_sstable_id_generator, Compactor, CompactorContext};
-    use crate::hummock::iterator::test_utils::mock_sstable_store;
-    use crate::hummock::HummockStorage;
-    use crate::monitor::{StateStoreMetrics, StoreLocalStatistic};
-    use crate::storage_value::StorageValue;
-    use crate::store::{ReadOptions, WriteOptions};
-    use crate::{Keyspace, StateStore};
+    use risingwave_storage::hummock::compaction_group_client::DummyCompactionGroupClient;
+    use risingwave_storage::hummock::compactor::{
+        get_remote_sstable_id_generator, Compactor, CompactorContext,
+    };
+    use risingwave_storage::hummock::iterator::test_utils::mock_sstable_store;
+    use risingwave_storage::hummock::HummockStorage;
+    use risingwave_storage::monitor::{StateStoreMetrics, StoreLocalStatistic};
+    use risingwave_storage::storage_value::StorageValue;
+    use risingwave_storage::store::{ReadOptions, WriteOptions};
+    use risingwave_storage::{Keyspace, StateStore};
 
     async fn get_hummock_storage(
         hummock_meta_client: Arc<dyn HummockMetaClient>,
@@ -102,7 +103,7 @@ mod tests {
             hummock_meta_client
                 .commit_epoch(
                     epoch,
-                    storage.local_version_manager.get_uncommitted_ssts(epoch),
+                    storage.local_version_manager().get_uncommitted_ssts(epoch),
                 )
                 .await
                 .unwrap();
@@ -374,7 +375,7 @@ mod tests {
             write_batch.ingest().await.unwrap();
 
             storage.sync(Some(epoch)).await.unwrap();
-            let ssts = storage.local_version_manager.get_uncommitted_ssts(epoch);
+            let ssts = storage.local_version_manager().get_uncommitted_ssts(epoch);
             hummock_meta_client.commit_epoch(epoch, ssts).await.unwrap();
         }
 
@@ -482,7 +483,7 @@ mod tests {
             hummock_meta_client
                 .commit_epoch(
                     epoch,
-                    storage.local_version_manager.get_uncommitted_ssts(epoch),
+                    storage.local_version_manager().get_uncommitted_ssts(epoch),
                 )
                 .await
                 .unwrap();
@@ -639,7 +640,7 @@ mod tests {
             hummock_meta_client
                 .commit_epoch(
                     *epoch,
-                    storage.local_version_manager.get_uncommitted_ssts(*epoch),
+                    storage.local_version_manager().get_uncommitted_ssts(*epoch),
                 )
                 .await
                 .unwrap();

src/storage/src/storage_failpoints/test_hummock.rs → src/storage/hummock_test/src/failpoint_tests.rs

@@ -19,14 +19,14 @@ use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_meta::hummock::test_utils::setup_compute_env;
 use risingwave_meta::hummock::MockHummockMetaClient;
 use risingwave_rpc_client::HummockMetaClient;
-
-use crate::hummock::compaction_group_client::DummyCompactionGroupClient;
-use crate::hummock::iterator::test_utils::mock_sstable_store;
-use crate::hummock::test_utils::{count_iter, default_config_for_test};
-use crate::hummock::HummockStorage;
-use crate::storage_value::StorageValue;
-use crate::store::{ReadOptions, WriteOptions};
-use crate::StateStore;
+use risingwave_storage::hummock::compaction_group_client::DummyCompactionGroupClient;
+use risingwave_storage::hummock::iterator::test_utils::mock_sstable_store;
+use risingwave_storage::hummock::test_utils::{count_iter, default_config_for_test};
+use risingwave_storage::hummock::HummockStorage;
+use risingwave_storage::monitor::StateStoreMetrics;
+use risingwave_storage::storage_value::StorageValue;
+use risingwave_storage::store::{ReadOptions, WriteOptions};
+use risingwave_storage::StateStore;
 
 #[tokio::test]
 #[cfg(all(test, feature = "failpoints"))]
@@ -46,7 +46,7 @@ async fn test_failpoints_state_store_read_upload() {
         hummock_options,
         sstable_store.clone(),
         meta_client.clone(),
-        Arc::new(crate::monitor::StateStoreMetrics::unused()),
+        Arc::new(StateStoreMetrics::unused()),
         Arc::new(DummyCompactionGroupClient::new(
             StaticCompactionGroupId::StateDefault.into(),
         )),

src/storage/hummock_test/src/lib.rs

@@ -0,0 +1,26 @@
+// Copyright 2022 Singularity Data
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(test)]
+mod compactor_tests;
+#[cfg(all(test, feature = "failpoints"))]
+mod failpoint_tests;
+#[cfg(test)]
+mod local_version_manager_tests;
+#[cfg(test)]
+mod snapshot_tests;
+#[cfg(test)]
+mod state_store_tests;
+#[cfg(test)]
+mod vacuum_tests;