Merge remote-tracking branch 'upstream/master' into patch-stack

Signed-off-by: Matt Klein <mklein@bitdrift.io>

.github/workflows/ci.yml

@@ -220,7 +220,6 @@ jobs:
         env:
             RUST_BACKTRACE: 1
             RUST_TEST_THREADS: 1
-            MIRIFLAGS: -Zmiri-tag-raw-pointers
         steps:
             - name: Checkout sources
               uses: actions/checkout@v2

CHANGELOG.md

@@ -1,8 +1,20 @@
 # Changelog
 
-## [3.3] - Unreleased
+## [3.5] - Unreleased
+
+- [Default to packed for repeated primitives in proto3](https://github.com/stepancheg/rust-protobuf/pull/707)
+- [Put correct flag in comment of generated files](https://github.com/stepancheg/rust-protobuf/pull/655)
+- [Option to disable non-exhausive annotation for oneof](https://github.com/stepancheg/rust-protobuf/pull/726)
+- [Option to generate `BTreeMap` for map fields](https://github.com/stepancheg/rust-protobuf/pull/700)
+- [Fix writing large messages](https://github.com/stepancheg/rust-protobuf/pull/725)
+
+## [3.4] - 2024-02-24
 
 * [Unnecessary copy in print_to_string_internal](https://github.com/stepancheg/rust-protobuf/pull/684)
+* [Ignore error of `flush` in `Drop` of `CodedOutputStream`](https://github.com/stepancheg/rust-protobuf/issues/714)
+* [Faster `encoded_varint64_len`](https://github.com/stepancheg/rust-protobuf/pull/709)
+* [`reserved` keyword in enums](https://github.com/stepancheg/rust-protobuf/pull/712)
+* [Set streaming options in pure parser](https://github.com/stepancheg/rust-protobuf/pull/646)
 
 ## [3.3.0] - 2023-09-30
 

LICENSE.txt

@@ -16,4 +16,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
-OR OTHER DEALINGS IN THE SOFTWARE.
+OR OTHER DEALINGS IN THE SOFTWARE.

ci-gen/src/main.rs

@@ -190,7 +190,7 @@ fn miri_test_job() -> Job {
     let env = vec![
         ("RUST_BACKTRACE".to_owned(), "1".to_owned()),
         ("RUST_TEST_THREADS".to_owned(), "1".to_owned()),
-        ("MIRIFLAGS".to_owned(), "-Zmiri-tag-raw-pointers".to_owned()),
+        // ("MIRIFLAGS".to_owned(), "-Zmiri-tag-raw-pointers".to_owned()),
     ];
     Job {
         id,

proto/rustproto.proto

@@ -19,6 +19,10 @@ extend google.protobuf.FileOptions {
     optional bool tokio_bytes_all = 17011;
     // Use `bytes::Bytes` for `string` fields
     optional bool tokio_bytes_for_string_all = 17012;
+    // When false, `#[non_exhaustive]` is not generated for `oneof` fields.
+    optional bool oneofs_non_exhaustive_all = 17013;
+    // When true, generate `BTreeMap` instead of `HashMap` for map fields.
+    optional bool btreemap_all = 17014;
 
     // When true, will only generate codes that works with lite runtime.
     optional bool lite_runtime_all = 17035;
@@ -33,6 +37,10 @@ extend google.protobuf.MessageOptions {
     optional bool tokio_bytes = 17011;
     // Use `bytes::Bytes` for `string` fields
     optional bool tokio_bytes_for_string = 17012;
+    // When false, `#[non_exhaustive]` is not generated for `oneof` fields.
+    optional bool oneofs_non_exhaustive = 17013;
+    // When true, generate `BTreeMap` instead of `HashMap` for map fields.
+    optional bool btreemap = 17014;
 }
 
 extend google.protobuf.FieldOptions {
@@ -44,4 +52,8 @@ extend google.protobuf.FieldOptions {
     optional bool tokio_bytes_field = 17011;
     // Use `bytes::Bytes` for `string` fields
     optional bool tokio_bytes_for_string_field = 17012;
+    // When false, `#[non_exhaustive]` is not generated for `oneof` fields.
+    optional bool oneofs_non_exhaustive_field = 17013;
+    // When true, generate `BTreeMap` instead of `HashMap` for map fields.
+    optional bool btreemap_field = 17014;
 }

protobuf-codegen/LICENSE.txt

@@ -16,4 +16,4 @@ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
-OR OTHER DEALINGS IN THE SOFTWARE.
+OR OTHER DEALINGS IN THE SOFTWARE.

protobuf-codegen/src/customize/mod.rs

@@ -98,6 +98,8 @@ pub struct Customize {
     pub(crate) tokio_bytes: Option<bool>,
     /// Use `bytes::Bytes` for `string` fields
     pub(crate) tokio_bytes_for_string: Option<bool>,
+    /// When false, `#[non_exhaustive]` is not generated for `oneof` fields.
+    pub(crate) oneofs_non_exhaustive: Option<bool>,
     /// Enable lite runtime.
     pub(crate) lite_runtime: Option<bool>,
     /// Generate `mod.rs` in the output directory.
@@ -109,6 +111,8 @@ pub struct Customize {
     /// Used internally to generate protos bundled in protobuf crate
     /// like `descriptor.proto`
     pub(crate) inside_protobuf: Option<bool>,
+    /// When true, protobuf maps are represented with `std::collections::BTreeMap`
+    pub(crate) btreemap: Option<bool>,
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -148,6 +152,11 @@ impl Customize {
         self
     }
 
+    pub fn oneofs_non_exhaustive(mut self, non_exhaustive: bool) -> Self {
+        self.oneofs_non_exhaustive = Some(non_exhaustive);
+        self
+    }
+
     /// Generate code for "lite runtime". Generated code contains no code for reflection.
     /// So the generated code (and more importantly, generated binary size) is smaller,
     /// but reflection, text format, JSON serialization won't work.
@@ -171,6 +180,14 @@ impl Customize {
         self
     }
 
+    /// Use btreemaps for maps representation
+    pub fn btreemaps(self, use_btreemaps: bool) -> Self {
+        Self {
+            btreemap: Some(use_btreemaps),
+            ..self
+        }
+    }
+
     /// Update fields of self with fields defined in other customize
     pub fn update_with(&mut self, that: &Customize) {
         if let Some(v) = &that.before {
@@ -197,6 +214,9 @@ impl Customize {
         if let Some(v) = that.inside_protobuf {
             self.inside_protobuf = Some(v);
         }
+        if let Some(v) = that.btreemap {
+            self.btreemap = Some(v);
+        }
     }
 
     /// Update unset fields of self with fields from other customize
@@ -236,6 +256,8 @@ impl Customize {
                 r.lite_runtime = Some(parse_bool(v)?);
             } else if n == "gen_mod_rs" {
                 r.gen_mod_rs = Some(parse_bool(v)?);
+            } else if n == "btreemap" {
+                r.btreemap = Some(parse_bool(v)?);
             } else if n == "inside_protobuf" {
                 r.inside_protobuf = Some(parse_bool(v)?);
             } else if n == "lite" {

protobuf-codegen/src/customize/rustproto_proto.rs

@@ -12,6 +12,8 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C
     let generate_getter = rustproto::exts::generate_getter.get(source);
     let tokio_bytes = rustproto::exts::tokio_bytes.get(source);
     let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string.get(source);
+    let oneofs_non_exhaustive = rustproto::exts::oneofs_non_exhaustive.get(source);
+    let btreemap = rustproto::exts::btreemap.get(source);
     let lite_runtime = None;
     let gen_mod_rs = None;
     let inside_protobuf = None;
@@ -21,9 +23,11 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C
         generate_getter,
         tokio_bytes,
         tokio_bytes_for_string,
+        oneofs_non_exhaustive,
         lite_runtime,
         gen_mod_rs,
         inside_protobuf,
+        btreemap,
     }
 }
 
@@ -37,6 +41,8 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo
     let generate_getter = rustproto::exts::generate_getter_field.get(source);
     let tokio_bytes = rustproto::exts::tokio_bytes_field.get(source);
     let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string_field.get(source);
+    let oneofs_non_exhaustive = rustproto::exts::oneofs_non_exhaustive_field.get(source);
+    let btreemap = rustproto::exts::btreemap_field.get(source);
     let lite_runtime = None;
     let gen_mod_rs = None;
     let inside_protobuf = None;
@@ -46,9 +52,11 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo
         generate_getter,
         tokio_bytes,
         tokio_bytes_for_string,
+        oneofs_non_exhaustive,
         lite_runtime,
         gen_mod_rs,
         inside_protobuf,
+        btreemap,
     }
 }
 
@@ -58,7 +66,9 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi
     let generate_getter = rustproto::exts::generate_getter_all.get(source);
     let tokio_bytes = rustproto::exts::tokio_bytes_all.get(source);
     let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string_all.get(source);
+    let oneofs_non_exhaustive = rustproto::exts::oneofs_non_exhaustive_all.get(source);
     let lite_runtime = rustproto::exts::lite_runtime_all.get(source);
+    let btreemap = rustproto::exts::btreemap_all.get(source);
     let gen_mod_rs = None;
     let inside_protobuf = None;
     Customize {
@@ -67,8 +77,10 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi
         generate_getter,
         tokio_bytes,
         tokio_bytes_for_string,
+        oneofs_non_exhaustive,
         lite_runtime,
         inside_protobuf,
         gen_mod_rs,
+        btreemap,
     }
 }

protobuf-codegen/src/gen/field/accessor.rs

@@ -65,8 +65,8 @@ impl FieldGen<'_> {
     fn accessor_fn_map(&self, map_field: &MapField) -> AccessorFn {
         let MapField { .. } = map_field;
         AccessorFn {
-            name: "make_map_simpler_accessor".to_owned(),
-            type_params: vec![format!("_"), format!("_")],
+            name: "make_map_simpler_accessor_new".to_owned(),
+            type_params: vec![format!("_")],
             callback_params: self.make_accessor_fns_lambda(),
         }
     }

protobuf-codegen/src/gen/field/mod.rs

@@ -117,10 +117,6 @@ impl<'a> SingularOrOneofField<'a> {
     }
 }
 
-// Representation of map entry: key type and value type
-#[derive(Clone, Debug)]
-pub struct EntryKeyValue<'a>(FieldElem<'a>, FieldElem<'a>);
-
 #[derive(Clone)]
 pub(crate) struct FieldGen<'a> {
     syntax: Syntax,
@@ -185,11 +181,43 @@ impl<'a> FieldGen<'a> {
             }
             RuntimeFieldType::Repeated(..) => {
                 let elem = field_elem(&field, root_scope, &customize);
-
-                FieldKind::Repeated(RepeatedField {
-                    elem,
-                    packed: field.field.proto().options.get_or_default().packed(),
-                })
+                let primitive = match field.field.proto().type_() {
+                    Type::TYPE_DOUBLE
+                    | Type::TYPE_FLOAT
+                    | Type::TYPE_INT64
+                    | Type::TYPE_UINT64
+                    | Type::TYPE_INT32
+                    | Type::TYPE_FIXED64
+                    | Type::TYPE_FIXED32
+                    | Type::TYPE_BOOL
+                    | Type::TYPE_UINT32
+                    | Type::TYPE_SFIXED32
+                    | Type::TYPE_SFIXED64
+                    | Type::TYPE_SINT32
+                    | Type::TYPE_SINT64
+                    | Type::TYPE_ENUM => true,
+                    Type::TYPE_STRING
+                    | Type::TYPE_GROUP
+                    | Type::TYPE_MESSAGE
+                    | Type::TYPE_BYTES => false,
+                };
+                let packed = field
+                    .field
+                    .proto()
+                    .options
+                    .get_or_default()
+                    .packed
+                    .unwrap_or(match field.message.scope.file_scope.syntax() {
+                        Syntax::Proto2 => false,
+                        // in proto3, repeated primitive types are packed by default
+                        Syntax::Proto3 => primitive,
+                    });
+                if packed && !primitive {
+                    anyhow::bail!(
+                        "[packed = true] can only be specified for repeated primitive fields"
+                    );
+                }
+                FieldKind::Repeated(RepeatedField { elem, packed })
             }
             RuntimeFieldType::Singular(..) => {
                 let elem = field_elem(&field, root_scope, &customize);
@@ -274,6 +302,12 @@ impl<'a> FieldGen<'a> {
     pub(crate) fn full_storage_type(&self, reference: &FileAndMod) -> RustType {
         match self.kind {
             FieldKind::Repeated(ref repeated) => repeated.rust_type(reference),
+            FieldKind::Map(MapField {
+                ref key, ref value, ..
+            }) if self.customize.btreemap == Some(true) => RustType::BTreeMap(
+                Box::new(key.rust_storage_elem_type(reference)),
+                Box::new(value.rust_storage_elem_type(reference)),
+            ),
             FieldKind::Map(MapField {
                 ref key, ref value, ..
             }) => RustType::HashMap(

protobuf-codegen/src/gen/field/type_ext.rs

@@ -5,7 +5,7 @@ use crate::gen::rust_types_values::RustType;
 
 pub(crate) trait TypeExt {
     fn read(&self, is: &str, primitive_type_variant: PrimitiveTypeVariant) -> String;
-    fn is_s_varint(&self) -> bool;
+    fn _is_s_varint(&self) -> bool;
     fn is_copy(&self) -> bool;
     fn protobuf_name(&self) -> &'static str;
     fn rust_type(&self) -> RustType;
@@ -28,7 +28,7 @@ impl TypeExt for Type {
     }
 
     /// True if self is signed integer with zigzag encoding
-    fn is_s_varint(&self) -> bool {
+    fn _is_s_varint(&self) -> bool {
         match *self {
             Type::TYPE_SINT32 | Type::TYPE_SINT64 => true,
             _ => false,

protobuf-codegen/src/gen/oneof.rs

@@ -252,6 +252,14 @@ impl<'a> OneofGen<'a> {
     fn write_enum(&self, w: &mut CodeWriter) {
         let derive = vec!["Clone", "PartialEq", "Debug"];
         w.derive(&derive);
+        if self
+            .customize
+            .for_elem
+            .oneofs_non_exhaustive
+            .unwrap_or(true)
+        {
+            w.write_line("#[non_exhaustive]");
+        }
         write_protoc_insertion_point_for_oneof(w, &self.customize.for_elem, &self.oneof.oneof);
         w.pub_enum(&self.oneof.rust_name().ident.to_string(), |w| {
             for variant in self.variants_except_group() {

protobuf-codegen/src/gen/rust_types_values.rs

@@ -34,6 +34,7 @@ pub(crate) enum RustType {
     Bool,
     Vec(Box<RustType>),
     HashMap(Box<RustType>, Box<RustType>),
+    BTreeMap(Box<RustType>, Box<RustType>),
     String,
     // [T], not &[T]
     Slice(Box<RustType>),
@@ -75,6 +76,11 @@ impl RustType {
                 key.to_code(customize),
                 value.to_code(customize)
             ),
+            RustType::BTreeMap(ref key, ref value) => format!(
+                "::std::collections::BTreeMap<{}, {}>",
+                key.to_code(customize),
+                value.to_code(customize)
+            ),
             RustType::String => format!("::std::string::String"),
             RustType::Slice(ref param) => format!("[{}]", param.to_code(customize)),
             RustType::Str => format!("str"),
@@ -214,17 +220,18 @@ impl RustType {
     // default value for type
     pub fn default_value(&self, customize: &Customize, const_expr: bool) -> String {
         match *self {
-            RustType::Ref(ref t) if t.is_str() => "\"\"".to_string(),
-            RustType::Ref(ref t) if t.is_slice().is_some() => "&[]".to_string(),
-            RustType::Int(..) => "0".to_string(),
-            RustType::Float(..) => "0.".to_string(),
-            RustType::Bool => "false".to_string(),
-            RustType::Vec(..) => EXPR_VEC_NEW.to_string(),
-            RustType::HashMap(..) => "::std::collections::HashMap::new()".to_string(),
-            RustType::String => "::std::string::String::new()".to_string(),
-            RustType::Bytes => "::bytes::Bytes::new()".to_string(),
+            RustType::Ref(ref t) if t.is_str() => "\"\"".to_owned(),
+            RustType::Ref(ref t) if t.is_slice().is_some() => "&[]".to_owned(),
+            RustType::Int(..) => "0".to_owned(),
+            RustType::Float(..) => "0.".to_owned(),
+            RustType::Bool => "false".to_owned(),
+            RustType::Vec(..) => EXPR_VEC_NEW.to_owned(),
+            RustType::HashMap(..) => "::std::collections::HashMap::new()".to_owned(),
+            RustType::BTreeMap(..) => "::std::collections::BTreeMap::new()".to_owned(),
+            RustType::String => "::std::string::String::new()".to_owned(),
+            RustType::Bytes => "::bytes::Bytes::new()".to_owned(),
             RustType::Chars => format!("{}::Chars::new()", protobuf_crate_path(customize)),
-            RustType::Option(..) => EXPR_NONE.to_string(),
+            RustType::Option(..) => EXPR_NONE.to_owned(),
             RustType::MessageField(..) => {
                 format!("{}::MessageField::none()", protobuf_crate_path(customize))
             }
@@ -266,7 +273,8 @@ impl RustType {
             | RustType::Chars
             | RustType::String
             | RustType::MessageField(..)
-            | RustType::HashMap(..) => format!("{}.clear()", v),
+            | RustType::HashMap(..)
+            | RustType::BTreeMap(..) => format!("{}.clear()", v),
             RustType::Bool
             | RustType::Float(..)
             | RustType::Int(..)

protobuf-codegen/src/protoc_gen_rust.rs

@@ -11,7 +11,7 @@ pub fn protoc_gen_rust_main() {
         let customize = Customize::parse_from_parameter(r.parameter).expect("parse options");
         gen_all(
             r.file_descriptors,
-            "protoc --rust-out=...",
+            "protoc --rust_out=...",
             r.files_to_generate,
             &customize,
             &CustomizeCallbackDefault,