vmm: support pod sysctl configuration

Signed-off-by: MorningTZH <morningtzh@yeah.net>

vmm/common/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 panic = 'abort'
 
 [dependencies]
-containerd-sandbox = {git="https://github.com/kuasar-io/rust-extensions.git"}
+containerd-sandbox = { git = "https://github.com/kuasar-io/rust-extensions.git" }
 serde = "1.0.139"
 lazy_static = "1.4.0"
 nix = "0.24.1"
@@ -20,4 +20,5 @@ async-trait = "0.1"
 regex = "1.5.6"
 
 [build-dependencies]
-ttrpc-codegen = "0.4"
+ttrpc-codegen = { git = "https://github.com/kuasar-io/ttrpc-rust.git", branch = "v0.7.1-kuasar" }
+tonic-build = "0.7.2"

vmm/common/src/protos/sandbox.proto

@@ -19,6 +19,7 @@ syntax = "proto3";
 package grpc;
 
 import "google/protobuf/empty.proto";
+import "google/protobuf/any.proto";
 import "github.com/containerd/containerd/api/services/ttrpc/events/v1/events.proto";
 
 service SandboxService {
@@ -31,6 +32,7 @@ service SandboxService {
     rpc ExecVMProcess (ExecVMProcessRequest) returns (ExecVMProcessResponse);
     rpc SyncClock (SyncClockPacket) returns (SyncClockPacket);
     rpc GetEvents (google.protobuf.Empty) returns (containerd.services.events.ttrpc.v1.Envelope);
+    rpc SetupSandbox (SetupSandboxRequest) returns (google.protobuf.Empty);
 }
 
 message CheckRequest {
@@ -108,4 +110,10 @@ message UpdateInterfacesRequest {
 
 message UpdateRoutesRequest {
     repeated Route routes = 1;
+}
+
+message SetupSandboxRequest {
+    google.protobuf.Any config = 1;
+    repeated Interface interfaces = 2;
+    repeated Route routes = 3;
 }

vmm/sandbox/src/client.rs

@@ -48,12 +48,10 @@ use ttrpc::{
     r#async::{Client, TtrpcContext},
 };
 use vmm_common::api::{
-    sandbox::{CheckRequest, SyncClockPacket, UpdateInterfacesRequest, UpdateRoutesRequest},
+    sandbox::{CheckRequest, SetupSandboxRequest, SyncClockPacket},
     sandbox_ttrpc::SandboxServiceClient,
 };
 
-use crate::network::{NetworkInterface, Route};
-
 const HVSOCK_RETRY_TIMEOUT_IN_MS: u64 = 10;
 // TODO: reduce to 10s
 const NEW_TTRPC_CLIENT_TIMEOUT: u64 = 45;
@@ -241,34 +239,17 @@ async fn do_check_agent(client: &SandboxServiceClient, timeout: u64) {
     }
 }
 
-pub(crate) async fn client_update_interfaces(
+pub(crate) async fn client_setup_sandbox(
     client: &SandboxServiceClient,
-    intfs: &[NetworkInterface],
+    config: &SetupSandboxRequest,
 ) -> Result<()> {
-    let mut req = UpdateInterfacesRequest::new();
-    req.interfaces = intfs.iter().map(|x| x.into()).collect();
-
     client
-        .update_interfaces(
+        .setup_sandbox(
             with_timeout(Duration::from_secs(10).as_nanos() as i64),
-            &req,
+            &config,
         )
         .await
-        .map_err(|e| anyhow!("failed to update interfaces: {}", e))?;
-    Ok(())
-}
-
-pub(crate) async fn client_update_routes(
-    client: &SandboxServiceClient,
-    rts: &[Route],
-) -> Result<()> {
-    let mut req = UpdateRoutesRequest::new();
-    req.routes = rts.iter().map(|x| x.into()).collect();
-
-    client
-        .update_routes(with_timeout(Duration::from_secs(3).as_nanos() as i64), &req)
-        .await
-        .map_err(|e| anyhow!("failed to update routes: {}", e))?;
+        .map_err(|e| anyhow!("failed to setup sandbox: {}", e))?;
     Ok(())
 }
 

vmm/sandbox/src/cloud_hypervisor/factory.rs

@@ -22,7 +22,6 @@ use crate::{
         devices::{console::Console, fs::Fs, pmem::Pmem, rng::Rng, vsock::Vsock},
         CloudHypervisorVM,
     },
-    sandbox::has_shared_pid_namespace,
     utils::get_netns,
     vm::VMFactory,
 };
@@ -47,9 +46,6 @@ impl VMFactory for CloudHypervisorVMFactory {
     ) -> containerd_sandbox::error::Result<Self::VM> {
         let netns = get_netns(&s.sandbox);
         let mut vm = CloudHypervisorVM::new(id, &netns, &s.base_dir, &self.vm_config);
-        if has_shared_pid_namespace(&s.sandbox) {
-            vm.config.cmdline.push_str(" task.share_pidns")
-        }
         // add image as a disk
         if !self.vm_config.common.image_path.is_empty() {
             let rootfs_device = Pmem::new("rootfs", &self.vm_config.common.image_path, true);

vmm/sandbox/src/sandbox.rs

@@ -28,6 +28,7 @@ use containerd_sandbox::{
 };
 use containerd_shim::{protos::api::Envelope, util::write_str_to_file};
 use log::{debug, error, info, warn};
+use protobuf::{well_known_types::any::Any, MessageField};
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use tokio::{
     fs::{copy, create_dir_all, remove_dir_all, OpenOptions},
@@ -36,17 +37,14 @@ use tokio::{
 };
 use ttrpc::context::with_timeout;
 use vmm_common::{
-    api::{empty::Empty, sandbox_ttrpc::SandboxServiceClient},
+    api::{empty::Empty, sandbox::SetupSandboxRequest, sandbox_ttrpc::SandboxServiceClient},
     storage::Storage,
     ETC_HOSTS, ETC_RESOLV, HOSTNAME_FILENAME, HOSTS_FILENAME, RESOLV_FILENAME, SHARED_DIR_SUFFIX,
 };
 
 use crate::{
     cgroup::{SandboxCgroup, DEFAULT_CGROUP_PARENT_PATH},
-    client::{
-        client_check, client_sync_clock, client_update_interfaces, client_update_routes,
-        new_sandbox_client,
-    },
+    client::{client_check, client_setup_sandbox, client_sync_clock, new_sandbox_client},
     container::KuasarContainer,
     network::{Network, NetworkConfig},
     utils::{get_dns_config, get_hostname, get_resources, get_sandbox_cgroup_parent_path},
@@ -451,9 +449,9 @@ where
             return Err(e);
         }
 
-        if let Err(e) = self.setup_network().await {
+        if let Err(e) = self.setup_sandbox().await {
             if let Err(re) = self.vm.stop(true).await {
-                warn!("roll back in init task client: {}", re);
+                error!("roll back in setup sandbox client: {}", re);
                 return Err(e);
             }
             return Err(e);
@@ -526,13 +524,37 @@ where
         Ok(())
     }
 
-    pub(crate) async fn setup_network(&mut self) -> Result<()> {
-        if let Some(network) = self.network.as_ref() {
-            if let Some(client) = &*self.client.lock().await {
-                client_update_interfaces(client, network.interfaces()).await?;
-                client_update_routes(client, network.routes()).await?;
+    pub(crate) async fn setup_sandbox(&mut self) -> Result<()> {
+        let mut req = SetupSandboxRequest::new();
+
+        if let Some(client) = &*self.client.lock().await {
+            // Set PodSandboxConfig
+            if let Some(config) = &self.data.config {
+                let config_str = serde_json::to_vec(config).map_err(|e| {
+                    Error::Other(anyhow!(
+                        "failed to marshal PodSandboxConfig to string, {:?}",
+                        e
+                    ))
+                })?;
+
+                let mut any = Any::new();
+                any.type_url = "PodSandboxConfig".to_string();
+                any.value = config_str;
+
+                req.config = MessageField::some(any);
+            }
+
+            if let Some(network) = self.network.as_ref() {
+                // Set interfaces
+                req.interfaces = network.interfaces().iter().map(|x| x.into()).collect();
+
+                // Set routes
+                req.routes = network.routes().iter().map(|x| x.into()).collect();
             }
+
+            client_setup_sandbox(client, &req).await?;
         }
+
         Ok(())
     }
 

vmm/task/Cargo.toml

@@ -35,5 +35,6 @@ tokio-vsock = "0.3.1"
 pin-project-lite = "0.2.7"
 ttrpc = { version = "0.7", features = ["async"] }
 
+containerd-sandbox = { git = "https://github.com/kuasar-io/rust-extensions.git" }
 containerd-shim = { git = "https://github.com/kuasar-io/rust-extensions.git", features = ["async"] }
 runc = { git = "https://github.com/kuasar-io/rust-extensions.git", features = ["async"] }

vmm/task/src/config.rs

@@ -20,7 +20,6 @@ use tokio::fs::read_to_string;
 const SHAREFS_TYPE: &str = "task.sharefs_type";
 const LOG_LEVEL: &str = "task.log_level";
 const TASK_DEBUG: &str = "task.debug";
-const SHARE_PIDNS: &str = "task.share_pidns";
 
 macro_rules! parse_cmdline {
     ($param:ident, $key:ident, $field:expr) => {
@@ -42,7 +41,6 @@ macro_rules! parse_cmdline {
 pub struct TaskConfig {
     pub(crate) sharefs_type: String,
     pub(crate) log_level: String,
-    pub(crate) share_pidns: bool,
     pub(crate) debug: bool,
 }
 
@@ -51,7 +49,6 @@ impl Default for TaskConfig {
         TaskConfig {
             sharefs_type: "9p".to_string(),
             log_level: "info".to_string(),
-            share_pidns: false,
             debug: false,
         }
     }
@@ -69,7 +66,6 @@ impl TaskConfig {
             parse_cmdline!(param, SHAREFS_TYPE, config.sharefs_type, String::from);
             parse_cmdline!(param, LOG_LEVEL, config.log_level, String::from);
             parse_cmdline!(param, TASK_DEBUG, config.debug);
-            parse_cmdline!(param, SHARE_PIDNS, config.share_pidns);
         }
         Ok(config)
     }