Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: implement Background service for databend #11751

Merged
merged 43 commits into from
Jun 16, 2023
Merged
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
9885aed
feat: background service patch 1(schema definitions)
ZhiHanZ May 31, 2023
961c508
feat: add kv api for background tasks
ZhiHanZ May 31, 2023
f98d954
feat: add background_task apis with unit-test
ZhiHanZ Jun 1, 2023
9fa7784
feat: add background job schema
ZhiHanZ Jun 6, 2023
48254db
save
ZhiHanZ Jun 6, 2023
6fd1a77
feat: add compaction background job
ZhiHanZ Jun 7, 2023
a6798fc
save
ZhiHanZ Jun 7, 2023
b6462eb
save
ZhiHanZ Jun 7, 2023
641c0c9
save
ZhiHanZ Jun 7, 2023
9ac85c1
save
ZhiHanZ Jun 7, 2023
b802e1f
add task logics
ZhiHanZ Jun 7, 2023
5307646
save
ZhiHanZ Jun 8, 2023
ea61c28
save
ZhiHanZ Jun 8, 2023
e1caabf
save
ZhiHanZ Jun 8, 2023
0487eb1
save
ZhiHanZ Jun 12, 2023
1264835
feat: refactor background apis
ZhiHanZ Jun 12, 2023
f29cf2b
feat: add apis
ZhiHanZ Jun 13, 2023
a19f1a5
save
ZhiHanZ Jun 13, 2023
2b8a2af
save
ZhiHanZ Jun 13, 2023
ca394c5
lint
ZhiHanZ Jun 14, 2023
728e86e
merge
ZhiHanZ Jun 14, 2023
1e4c436
license
ZhiHanZ Jun 14, 2023
0b889e8
license
ZhiHanZ Jun 14, 2023
0eb6062
license
ZhiHanZ Jun 14, 2023
ecd2c78
fix debug settings
ZhiHanZ Jun 14, 2023
0b8ef38
Merge branch 'main' into background_service_zhihanz
ZhiHanZ Jun 14, 2023
2ad3405
fix comment
ZhiHanZ Jun 14, 2023
47ec195
Merge branch 'main' into background_service_zhihanz
ZhiHanZ Jun 15, 2023
bdd0a2b
fix
ZhiHanZ Jun 15, 2023
699b102
Merge branch 'main' into background_service_zhihanz
ZhiHanZ Jun 15, 2023
c181979
remove unneeded stats
ZhiHanZ Jun 15, 2023
1e09218
fix comment
ZhiHanZ Jun 15, 2023
122b648
Merge branch 'main' into background_service_zhihanz
ZhiHanZ Jun 15, 2023
7c95c4f
fix unit tests
ZhiHanZ Jun 15, 2023
11c3434
Merge branch 'main' into background_service_zhihanz
ZhiHanZ Jun 15, 2023
2572381
Merge branch 'main' into background_service_zhihanz
ZhiHanZ Jun 16, 2023
9793c2a
Merge branch 'main' into background_service_zhihanz
ZhiHanZ Jun 16, 2023
910967b
fix
ZhiHanZ Jun 16, 2023
9db185f
fix comment
ZhiHanZ Jun 16, 2023
9651574
remove unneeded kv call
ZhiHanZ Jun 16, 2023
6c94e45
fix
ZhiHanZ Jun 16, 2023
9295298
f
ZhiHanZ Jun 16, 2023
259cef7
Merge branch 'main' into background_service_zhihanz
ZhiHanZ Jun 16, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 35 additions & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -70,6 +70,7 @@ members = [
"src/query/storages/result_cache",
"src/query/users",
"src/query/ee-features/vacuum-handler",
"src/query/ee-features/background-service",
"src/query/ee-features/aggregating-index",
"src/query/ee-features/data-mask",
"src/query/ee-features/table-lock",
2 changes: 1 addition & 1 deletion src/binaries/Cargo.toml
Original file line number Diff line number Diff line change
@@ -57,9 +57,9 @@ databend-meta = { path = "../meta/service" }
databend-query = { path = "../query/service" }
enterprise-query = { path = "../query/ee" }
# enterprise-meta = { path = "../meta/ee" }
background-service = { path = "../query/ee-features/background-service" }
sharing-endpoint = { path = "../query/sharing-endpoint" }
storages-common-table-meta = { path = "../query/storages/common/table-meta" }

# Crates.io dependencies
anyerror = { workspace = true }
anyhow = { workspace = true }
19 changes: 17 additions & 2 deletions src/binaries/query/entry.rs
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@

use std::env;

use background_service::get_background_service_handler;
use common_base::mem_allocator::GlobalAllocator;
use common_base::runtime::GLOBAL_MEM_STAT;
use common_base::set_alloc_error_hook;
@@ -93,7 +94,7 @@ pub async fn init_services(conf: &InnerConfig) -> Result<()> {
GlobalServices::init(conf.clone()).await
}

pub async fn start_services(conf: &InnerConfig) -> Result<()> {
async fn precheck_services(conf: &InnerConfig) -> Result<()> {
if conf.query.max_memory_limit_enabled {
let size = conf.query.max_server_memory_usage as i64;
info!("Set memory limit: {}", size);
@@ -125,6 +126,11 @@ pub async fn start_services(conf: &InnerConfig) -> Result<()> {

#[cfg(not(target_os = "macos"))]
check_max_open_files();
Ok(())
}

pub async fn start_services(conf: &InnerConfig) -> Result<()> {
precheck_services(conf).await?;

let mut shutdown_handle = ShutdownHandle::create()?;

@@ -333,7 +339,16 @@ pub async fn start_services(conf: &InnerConfig) -> Result<()> {
}

info!("Ready for connections.");
shutdown_handle.wait_for_termination_request().await;
if conf.background.enable {
println!("Start background service");
get_background_service_handler()
.start(&mut shutdown_handle)
.await?;
// for one shot background service, we need to drop it manually.
drop(shutdown_handle);
} else {
shutdown_handle.wait_for_termination_request().await;
}
info!("Shutdown server.");
Ok(())
}
5 changes: 4 additions & 1 deletion src/common/exception/src/exception_code.rs
Original file line number Diff line number Diff line change
@@ -177,7 +177,10 @@ build_exceptions! {
/// LicenseKeyInvalid is used when license key verification error occurs
///
/// For example: license key is expired
LicenseKeyInvalid(1402)
LicenseKeyInvalid(1402),

BackgroundJobAlreadyExists(1501),
UnknownBackgroundJob(1502)
}

// Meta service errors [2001, 3000].
80 changes: 80 additions & 0 deletions src/meta/api/src/background_api.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Copyright 2021 Datafuse Labs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use common_meta_app::background::BackgroundJobInfo;
use common_meta_app::background::BackgroundTaskInfo;
use common_meta_app::background::CreateBackgroundJobReply;
use common_meta_app::background::CreateBackgroundJobReq;
use common_meta_app::background::DeleteBackgroundJobReply;
use common_meta_app::background::DeleteBackgroundJobReq;
use common_meta_app::background::GetBackgroundJobReply;
use common_meta_app::background::GetBackgroundJobReq;
use common_meta_app::background::GetBackgroundTaskReply;
use common_meta_app::background::GetBackgroundTaskReq;
use common_meta_app::background::ListBackgroundJobsReq;
use common_meta_app::background::ListBackgroundTasksReq;
use common_meta_app::background::UpdateBackgroundJobParamsReq;
use common_meta_app::background::UpdateBackgroundJobReply;
use common_meta_app::background::UpdateBackgroundJobStatusReq;
use common_meta_app::background::UpdateBackgroundTaskReply;
use common_meta_app::background::UpdateBackgroundTaskReq;

use crate::kv_app_error::KVAppError;

#[async_trait::async_trait]
pub trait BackgroundApi: Send + Sync {
async fn create_background_job(
&self,
req: CreateBackgroundJobReq,
) -> Result<CreateBackgroundJobReply, KVAppError>;

async fn drop_background_job(
&self,
req: DeleteBackgroundJobReq,
) -> Result<DeleteBackgroundJobReply, KVAppError>;

async fn update_background_job_status(
&self,
req: UpdateBackgroundJobStatusReq,
) -> Result<UpdateBackgroundJobReply, KVAppError>;

async fn update_background_job_params(
&self,
req: UpdateBackgroundJobParamsReq,
) -> Result<UpdateBackgroundJobReply, KVAppError>;

async fn get_background_job(
&self,
req: GetBackgroundJobReq,
) -> Result<GetBackgroundJobReply, KVAppError>;
async fn list_background_jobs(
&self,
req: ListBackgroundJobsReq,
) -> Result<Vec<(u64, BackgroundJobInfo)>, KVAppError>;
// Return a list of background tasks (task_id, BackgroundInfo)
async fn list_background_tasks(
&self,
req: ListBackgroundTasksReq,
) -> Result<Vec<(u64, BackgroundTaskInfo)>, KVAppError>;

async fn update_background_task(
&self,
req: UpdateBackgroundTaskReq,
) -> Result<UpdateBackgroundTaskReply, KVAppError>;

async fn get_background_task(
&self,
req: GetBackgroundTaskReq,
) -> Result<GetBackgroundTaskReply, KVAppError>;
}
Loading