Merge pull request #98 from mxpv/codegen

Remove generated code from shim-protos (and fix CI)

crates/shim-protos/Cargo.toml

@@ -13,10 +13,10 @@ homepage = "https://containerd.io"
 [dependencies]
 protobuf = "3.1"
 ttrpc = "0.7"
-async-trait = { version = "0.1.48", optional = true}
+async-trait = { version = "0.1.48", optional = true }
 
 [build-dependencies]
-ttrpc-codegen = { version = "0.4", optional = true }
+ttrpc-codegen = "0.4"
 
 [dev-dependencies]
 ctrlc = { version = "3.0", features = ["termination"] }
@@ -27,11 +27,6 @@ tokio = { version = "1.18", features = ["full"] }
 [features]
 default = []
 async = ["ttrpc/async", "async-trait"]
-# In current implementation we check in all generated TTRPC code because of the following Rust limitations:
-# - `include!` doesn't handle well .rs files with attributes: https://github.com/rust-lang/rust/issues/18810
-# - `cargo publish` will complain on modified `src` directory if it's modified from `build.rs` - https://github.com/rust-lang/cargo/issues/5073
-# - There is now way to include the whole module from `OUT_DIR`, not just individual files.
-generate_bindings = ["ttrpc-codegen", "async-trait", "ttrpc/async"]
 
 [[example]]
 name = "shim-proto-server"

crates/shim-protos/README.md

@@ -13,13 +13,6 @@ The `containerd-shim-protos` crate provides [Protobuf](https://github.com/protoc
 and [TTRPC](https://github.com/containerd/ttrpc.git) service definitions for the
 [Containerd shim v2](https://github.com/containerd/containerd/blob/main/runtime/v2/task/shim.proto) protocol.
 
-The message and service definitions are auto-generated from protobuf source files under `vendor/`
-by using [ttrpc-codegen](https://github.com/containerd/ttrpc-rust/tree/master/ttrpc-codegen). So please do not
-edit those auto-generated source files. If upgrading/modification is needed, please follow the steps:
-- Synchronize the latest protobuf source files from the upstream projects into directory 'vendor/'.
-- Re-generate the source files by `cargo build --features=generate_bindings`.
-- Commit the synchronized protobuf source files and auto-generated source files, keeping them in synchronization.
-
 ## Usage
 Add `containerd-shim-client` as a dependency in your `Cargo.toml`
 

crates/shim-protos/build.rs

@@ -14,34 +14,36 @@
    limitations under the License.
 */
 
-#![allow(unused_imports)]
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+use std::path::PathBuf;
+use std::{env, fs};
 
-use std::fs;
-use std::io::Write;
-use std::path::Path;
-#[cfg(feature = "generate_bindings")]
 use ttrpc_codegen::{Codegen, Customize, ProtobufCustomize};
 
-#[cfg(not(feature = "generate_bindings"))]
-fn main() {}
-
-#[cfg(feature = "generate_bindings")]
 fn main() {
-    codegen(
-        "src/cgroups",
+    genmodule(
+        "types",
         &[
             "vendor/gogoproto/gogo.proto",
-            "vendor/github.com/containerd/cgroups/stats/v1/metrics.proto",
+            "vendor/google/protobuf/empty.proto",
+            "vendor/github.com/containerd/containerd/protobuf/plugin/fieldpath.proto",
+            "vendor/github.com/containerd/containerd/api/types/mount.proto",
+            "vendor/github.com/containerd/containerd/api/types/task/task.proto",
         ],
-        true,
         false,
     );
 
-    codegen(
-        "src/events",
+    genmodule(
+        "cgroups",
+        &["vendor/github.com/containerd/cgroups/stats/v1/metrics.proto"],
+        false,
+    );
+
+    genmodule(
+        "events",
         &[
-            "vendor/gogoproto/gogo.proto",
-            "vendor/github.com/containerd/containerd/protobuf/plugin/fieldpath.proto",
+            "vendor/github.com/containerd/containerd/api/types/mount.proto",
             "vendor/github.com/containerd/containerd/api/events/container.proto",
             "vendor/github.com/containerd/containerd/api/events/content.proto",
             "vendor/github.com/containerd/containerd/api/events/image.proto",
@@ -50,73 +52,90 @@ fn main() {
             "vendor/github.com/containerd/containerd/api/events/task.proto",
         ],
         false,
-        false,
     );
 
-    // generate async service
-    codegen(
-        "src/shim",
-        &[
-            "vendor/gogoproto/gogo.proto",
-            "vendor/github.com/containerd/containerd/protobuf/plugin/fieldpath.proto",
-            "vendor/github.com/containerd/containerd/runtime/v2/task/shim.proto",
-            "vendor/github.com/containerd/containerd/api/services/ttrpc/events/v1/events.proto",
-        ],
-        false,
-        true,
-    );
-    fs::rename("src/shim/shim_ttrpc.rs", "src/shim/shim_ttrpc_async.rs").unwrap();
-    fs::rename("src/shim/events_ttrpc.rs", "src/shim/events_ttrpc_async.rs").unwrap();
-
-    codegen(
-        "src/shim",
+    genmodule(
+        "shim",
         &[
             "vendor/github.com/containerd/containerd/runtime/v2/runc/options/oci.proto",
             "vendor/github.com/containerd/containerd/runtime/v2/task/shim.proto",
             "vendor/github.com/containerd/containerd/api/services/ttrpc/events/v1/events.proto",
         ],
         false,
-        false,
     );
 
-    codegen(
-        "src/types",
-        &[
-            "vendor/gogoproto/gogo.proto",
-            "vendor/github.com/containerd/containerd/api/types/mount.proto",
-            "vendor/github.com/containerd/containerd/api/types/task/task.proto",
-            "vendor/google/protobuf/empty.proto",
-        ],
-        true,
-        false,
-    );
+    #[cfg(feature = "async")]
+    {
+        genmodule(
+            "shim_async",
+            &[
+                "vendor/github.com/containerd/containerd/runtime/v2/task/shim.proto",
+                "vendor/github.com/containerd/containerd/api/services/ttrpc/events/v1/events.proto",
+            ],
+            true,
+        );
+    }
 }
 
-#[cfg(feature = "generate_bindings")]
-fn codegen(
-    path: impl AsRef<Path>,
-    inputs: impl IntoIterator<Item = impl AsRef<Path>>,
-    gen_mod_rs: bool,
-    async_all: bool,
-) {
-    let path = path.as_ref();
+fn genmodule(name: &str, inputs: &[&str], async_all: bool) {
+    let mut out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    out_path.push(name);
 
-    fs::create_dir_all(&path).unwrap();
+    fs::create_dir_all(&out_path).unwrap();
 
     Codegen::new()
         .inputs(inputs)
         .include("vendor/")
         .rust_protobuf()
         .rust_protobuf_customize(
             ProtobufCustomize::default()
-                .gen_mod_rs(gen_mod_rs)
+                .gen_mod_rs(true)
                 .generate_accessors(true),
         )
         .customize(Customize {
             async_all,
             ..Default::default()
         })
-        .out_dir(path)
+        .out_dir(&out_path)
         .run()
         .expect("Failed to generate protos");
+
+    // Find all *.rs files generated by TTRPC codegen
+    let files = fs::read_dir(&out_path)
+        .unwrap()
+        .filter_map(|entry| {
+            let entry = entry.unwrap();
+            if !entry.file_type().unwrap().is_file() {
+                None
+            } else {
+                Some(entry.path())
+            }
+        })
+        .collect::<Vec<_>>();
+
+    // `include!` doesn't handle files with attributes:
+    // - https://github.com/rust-lang/rust/issues/18810
+    // - https://github.com/rust-lang/rfcs/issues/752
+    // Remove all lines that start with:
+    // - #![allow(unknown_lints)]
+    // - #![cfg_attr(rustfmt, rustfmt::skip)]
+    //
+    for path in files {
+        let file = File::open(&path).unwrap();
+
+        let joined = BufReader::new(file)
+            .lines()
+            .filter_map(|line| {
+                let line = line.unwrap();
+                if line.starts_with("#!") || line.starts_with("//!") {
+                    None
+                } else {
+                    Some(line)
+                }
+            })
+            .collect::<Vec<_>>()
+            .join("\r\n");
+
+        fs::write(&path, joined).unwrap();
+    }
 }

crates/shim-protos/src/cgroups.rs

@@ -0,0 +1,23 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+pub mod metrics {
+    include!(concat!(env!("OUT_DIR"), "/cgroups/metrics.rs"));
+}
+
+mod gogo {
+    pub use crate::types::gogo::*;
+}