fix the required + nullable issue

src/AutoRest.CSharp/Common/AutoRest/Plugins/Configuration.cs

@@ -49,6 +49,7 @@ public static class Options
             public const string PublicDiscriminatorProperty = "public-discriminator-property";
             public const string ShouldTreatBase64AsBinaryData = "should-treat-base64-as-binary-data";
             public const string MethodsToKeepClientDefaultValue = "methods-to-keep-client-default-value";
+            public const string DeserializeNullCollectionAsNullValue = "deserialize-null-collection-as-null-value";
             public const string UseCoreDataFactoryReplacements = "use-core-datafactory-replacements";
         }
 
@@ -76,6 +77,7 @@ public static void Initialize(
             bool disablePaginationTopRenaming,
             bool generateModelFactory,
             bool publicDiscriminatorProperty,
+            bool deserializeNullCollectionAsNullValue,
             bool useCoreDataFactoryReplacements,
             IReadOnlyList<string> modelFactoryForHlc,
             UnreferencedTypesHandlingOption unreferencedTypesHandling,
@@ -106,6 +108,7 @@ public static void Initialize(
             SingleTopLevelClient = singleTopLevelClient;
             GenerateModelFactory = generateModelFactory;
             PublicDiscriminatorProperty = publicDiscriminatorProperty;
+            DeserializeNullCollectionAsNullValue = deserializeNullCollectionAsNullValue;
             UnreferencedTypesHandling = unreferencedTypesHandling;
             UseOverloadsBetweenProtocolAndConvenience = useOverloadsBetweenProtocolAndConvenience;
             KeepNonOverloadableProtocolSignature = keepNonOverloadableProtocolSignature;
@@ -237,6 +240,12 @@ public static void Initialize(
         /// If true, the discriminator property will be public. If false (default), the discriminator property will be internal.
         /// </summary>
         public static bool PublicDiscriminatorProperty { get; private set; }
+
+        /// <summary>
+        /// Whether we should deserialize null collections in the payload as null values if this sets to true.
+        /// Default value is false, where we will construct an empty collection (ChangeTrackingList or ChangeTrackingDictionary) if we get null value for collections in the payload
+        /// </summary>
+        public static bool DeserializeNullCollectionAsNullValue { get; private set; }
         public static bool UseOverloadsBetweenProtocolAndConvenience { get; private set; }
         public static bool KeepNonOverloadableProtocolSignature { get; private set; }
 
@@ -288,6 +297,7 @@ public static void Initialize(IPluginCommunication autoRest, string defaultNames
                 disablePaginationTopRenaming: GetOptionBoolValue(autoRest, Options.DisablePaginationTopRenaming),
                 generateModelFactory: GetOptionBoolValue(autoRest, Options.GenerateModelFactory),
                 publicDiscriminatorProperty: GetOptionBoolValue(autoRest, Options.PublicDiscriminatorProperty),
+                deserializeNullCollectionAsNullValue: GetOptionBoolValue(autoRest, Options.DeserializeNullCollectionAsNullValue),
                 modelFactoryForHlc: autoRest.GetValue<string[]?>(Options.ModelFactoryForHlc).GetAwaiter().GetResult() ?? Array.Empty<string>(),
                 unreferencedTypesHandling: GetOptionEnumValue<UnreferencedTypesHandlingOption>(autoRest, Options.UnreferencedTypesHandling),
                 useOverloadsBetweenProtocolAndConvenience: GetOptionBoolValue(autoRest, Options.UseOverloadsBetweenProtocolAndConvenience),
@@ -367,6 +377,8 @@ private static bool GetOptionBoolValue(IPluginCommunication autoRest, string opt
                     return false;
                 case Options.ShouldTreatBase64AsBinaryData:
                     return true;
+                case Options.DeserializeNullCollectionAsNullValue:
+                    return false;
                 case Options.UseCoreDataFactoryReplacements:
                     return true;
                 default:
@@ -438,6 +450,7 @@ internal static void LoadConfiguration(JsonElement root, string? projectPath, st
                 ReadOption(root, Options.DisablePaginationTopRenaming),
                 ReadOption(root, Options.GenerateModelFactory),
                 ReadOption(root, Options.PublicDiscriminatorProperty),
+                ReadOption(root, Options.DeserializeNullCollectionAsNullValue),
                 ReadOption(root, Options.UseCoreDataFactoryReplacements),
                 oldModelFactoryEntries,
                 ReadEnumOption<UnreferencedTypesHandlingOption>(root, Options.UnreferencedTypesHandling),

src/AutoRest.CSharp/Common/Generation/Writers/CodeWriterExtensions.cs

@@ -5,6 +5,8 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Text.Json;
+using AutoRest.CSharp.Common.Output.Models;
 using AutoRest.CSharp.Generation.Types;
 using AutoRest.CSharp.Output.Models;
 using AutoRest.CSharp.Output.Models.Requests;
@@ -15,11 +17,9 @@
 using AutoRest.CSharp.Output.Models.Types;
 using AutoRest.CSharp.Utilities;
 using Azure;
-using AutoRest.CSharp.Common.Output.Models;
 using Azure.Core;
-using static AutoRest.CSharp.Output.Models.MethodSignatureModifiers;
-using System.Text.Json;
 using Azure.Core.Pipeline;
+using static AutoRest.CSharp.Output.Models.MethodSignatureModifiers;
 
 namespace AutoRest.CSharp.Generation.Writers
 {
@@ -143,7 +143,7 @@ public static CodeWriter WriteFieldDeclarations(this CodeWriter writer, IEnumera
 
         public static IDisposable WriteMethodDeclaration(this CodeWriter writer, MethodSignatureBase methodBase, params string[] disabledWarnings)
         {
-            if (methodBase.Attributes is {} attributes)
+            if (methodBase.Attributes is { } attributes)
             {
                 foreach (var attribute in attributes)
                 {
@@ -620,6 +620,9 @@ internal static string GetConversion(CodeWriter writer, CSharpType from, CSharpT
             }
 
             var propertyName = property.PropertyName;
+            if (TypeFactory.IsCollectionType(property.ValueType) && property.IsRequired)
+                return writer.Scope($"if ({propertyName} != null && {typeof(Optional)}.{nameof(Optional.IsCollectionDefined)}({propertyName}))");
+
             return writer.Scope($"if ({propertyName} != null)");
         }
 

src/AutoRest.CSharp/Common/Generation/Writers/JsonCodeWriterExtensions.cs

@@ -365,7 +365,24 @@ private static void DeserializeIntoObjectProperties(this CodeWriter writer, IEnu
                         var emptyStringCheck = GetEmptyStringCheckClause(property, itemVariable, shouldTreatEmptyStringAsNull);
                         using (writer.Scope($"if ({itemVariable}.Value.ValueKind == {typeof(JsonValueKind)}.Null{emptyStringCheck})"))
                         {
-                            writer.Line($"{propertyVariables[property].Declaration} = null;");
+                            if (Configuration.DeserializeNullCollectionAsNullValue)
+                            {
+                                // we will assign null to everything if we have this configuration on
+                                writer.Line($"{propertyVariables[property].Declaration} = null;");
+                            }
+                            else
+                            {
+                                // we only assign null when it is not a collection if we have this configuration off
+                                if (!TypeFactory.IsCollectionType(property.ValueType))
+                                {
+                                    writer.Line($"{propertyVariables[property].Declaration} = null;");
+                                }
+                                else if (property.IsRequired)
+                                {
+                                    // specially when it is required, we assign ChangeTrackingList because for optional lists we are already doing that
+                                    writer.Line($"{propertyVariables[property].Declaration} = new {TypeFactory.GetPropertyImplementationType(property.ValueType)}();");
+                                }
+                            }
                             writer.Append($"continue;");
                         }
                     }

src/AutoRest.CSharp/Common/Input/CodeModelConverter.cs

@@ -246,7 +246,8 @@ private InputModelType GetOrCreateModel(ObjectSchema schema, SchemaUsageProvider
                     : null,
                 DerivedModels: derived,
                 DiscriminatorValue: schema.DiscriminatorValue,
-                DiscriminatorPropertyName: schema.Discriminator?.Property.SerializedName);
+                DiscriminatorPropertyName: schema.Discriminator?.Property.SerializedName,
+                IsNullable: false);
 
             _modelsCache[schema] = model;
             _modelPropertiesCache[schema] = properties;
@@ -380,8 +381,8 @@ private static InputType CreateType(Schema schema, string? format, IReadOnlyDict
             ChoiceSchema choiceSchema => CreateEnumType(choiceSchema, choiceSchema.ChoiceType, choiceSchema.Choices, true),
             SealedChoiceSchema choiceSchema => CreateEnumType(choiceSchema, choiceSchema.ChoiceType, choiceSchema.Choices, false),
 
-            ArraySchema array when IsDPG => new InputListType(array.Name, CreateType(array.ElementType, modelsCache, array.NullableItems ?? false)),
-            DictionarySchema dictionary when IsDPG => new InputDictionaryType(dictionary.Name, InputPrimitiveType.String, CreateType(dictionary.ElementType, modelsCache, dictionary.NullableItems ?? false)),
+            ArraySchema array when IsDPG => new InputListType(array.Name, CreateType(array.ElementType, modelsCache, array.NullableItems ?? false), false),
+            DictionarySchema dictionary when IsDPG => new InputDictionaryType(dictionary.Name, InputPrimitiveType.String, CreateType(dictionary.ElementType, modelsCache, dictionary.NullableItems ?? false), false),
             ObjectSchema objectSchema when IsDPG && modelsCache != null => modelsCache[objectSchema],
 
             AnySchema when IsDPG => InputIntrinsicType.Unknown,
@@ -411,7 +412,7 @@ private static InputLiteralType CreateLiteralType(ConstantSchema constantSchema,
                 _ => rawValue
             };
 
-            return new InputLiteralType("Literal", valueType, normalizedValue);
+            return new InputLiteralType("Literal", valueType, normalizedValue, false);
         }
 
         public static InputEnumType CreateEnumType(Schema schema, PrimitiveSchema choiceType, IEnumerable<ChoiceValue> choices, bool isExtensible) => new(

src/AutoRest.CSharp/Common/Input/InputTypes/InputDictionaryType.cs

@@ -3,4 +3,4 @@
 
 namespace AutoRest.CSharp.Common.Input;
 
-internal record InputDictionaryType(string Name, InputType KeyType, InputType ValueType, bool IsNullable = false) : InputType(Name, IsNullable) { }
+internal record InputDictionaryType(string Name, InputType KeyType, InputType ValueType, bool IsNullable) : InputType(Name, IsNullable) { }

src/AutoRest.CSharp/Common/Input/InputTypes/InputListType.cs

@@ -3,4 +3,4 @@
 
 namespace AutoRest.CSharp.Common.Input;
 
-internal record InputListType(string Name, InputType ElementType, bool IsNullable = false) : InputType(Name, IsNullable) { }
+internal record InputListType(string Name, InputType ElementType, bool IsNullable) : InputType(Name, IsNullable) { }

src/AutoRest.CSharp/Common/Input/InputTypes/InputLiteralType.cs

@@ -3,4 +3,4 @@
 
 namespace AutoRest.CSharp.Common.Input;
 
-internal record InputLiteralType(string Name, InputType LiteralValueType, object Value, bool IsNullable = false) : InputType(Name, IsNullable);
+internal record InputLiteralType(string Name, InputType LiteralValueType, object Value, bool IsNullable) : InputType(Name, IsNullable);

src/AutoRest.CSharp/Common/Input/InputTypes/InputModelType.cs

@@ -5,8 +5,8 @@
 
 namespace AutoRest.CSharp.Common.Input
 {
-    internal record InputModelType(string Name, string? Namespace, string? Accessibility, string? Deprecated, string? Description, InputModelTypeUsage Usage, IReadOnlyList<InputModelProperty> Properties, InputModelType? BaseModel, IReadOnlyList<InputModelType> DerivedModels, string? DiscriminatorValue, string? DiscriminatorPropertyName)
-        : InputType(Name)
+    internal record InputModelType(string Name, string? Namespace, string? Accessibility, string? Deprecated, string? Description, InputModelTypeUsage Usage, IReadOnlyList<InputModelProperty> Properties, InputModelType? BaseModel, IReadOnlyList<InputModelType> DerivedModels, string? DiscriminatorValue, string? DiscriminatorPropertyName, bool IsNullable)
+        : InputType(Name, IsNullable)
     {
         /// <summary>
         /// Indicates if this model is the Unknown derived version of a model with discriminator

src/AutoRest.CSharp/Common/Input/InputTypes/InputParameter.cs

@@ -31,7 +31,7 @@ public InputParameter() : this(
         Name: string.Empty,
         NameInRequest: string.Empty,
         Description: null,
-        Type: new InputPrimitiveType(InputTypeKind.Object),
+        Type: new InputPrimitiveType(InputTypeKind.Object, false),
         Location: RequestLocation.None,
         DefaultValue: null,
         VirtualParameter: null,

src/AutoRest.CSharp/Common/Input/InputTypes/InputPrimitiveType.cs

@@ -3,8 +3,10 @@
 
 namespace AutoRest.CSharp.Common.Input;
 
-internal record InputPrimitiveType(InputTypeKind Kind, bool IsNullable = false) : InputType(Kind.ToString(), IsNullable)
+internal record InputPrimitiveType(InputTypeKind Kind, bool IsNullable) : InputType(Kind.ToString(), IsNullable)
 {
+    private InputPrimitiveType(InputTypeKind kind) : this(kind, false) { }
+
     public static InputPrimitiveType AzureLocation { get; } = new(InputTypeKind.AzureLocation);
     public static InputPrimitiveType BinaryData { get; } = new(InputTypeKind.BinaryData);
     public static InputPrimitiveType Boolean { get; } = new(InputTypeKind.Boolean);

src/AutoRest.CSharp/Common/Input/InputTypes/InputType.cs

@@ -3,4 +3,4 @@
 
 namespace AutoRest.CSharp.Common.Input;
 
-internal abstract record InputType(string Name, bool IsNullable = false) { }
+internal abstract record InputType(string Name, bool IsNullable) { }

src/AutoRest.CSharp/Common/Input/InputTypes/InputUnionType.cs

@@ -5,4 +5,4 @@
 
 namespace AutoRest.CSharp.Common.Input;
 
-internal record InputUnionType(string Name, IReadOnlyList<InputType> UnionItemTypes, bool IsNullable = false) : InputType(Name, IsNullable);
+internal record InputUnionType(string Name, IReadOnlyList<InputType> UnionItemTypes, bool IsNullable) : InputType(Name, IsNullable);

src/AutoRest.CSharp/Common/Input/InputTypes/Serialization/TypeSpecInputDictionaryTypeConverter.cs

@@ -25,12 +25,14 @@ public override void Write(Utf8JsonWriter writer, InputDictionaryType value, Jso
         public static InputDictionaryType CreateDictionaryType(ref Utf8JsonReader reader, string? id, string? name, JsonSerializerOptions options)
         {
             var isFirstProperty = id == null && name == null;
+            bool isNullable = false;
             InputType? keyType = null;
             InputType? valueType = null;
             while (reader.TokenType != JsonTokenType.EndObject)
             {
                 var isKnownProperty = reader.TryReadReferenceId(ref isFirstProperty, ref id)
-                    || reader.TryReadString(nameof(InputListType.Name), ref name)
+                    || reader.TryReadString(nameof(InputDictionaryType.Name), ref name)
+                    || reader.TryReadBoolean(nameof(InputDictionaryType.IsNullable), ref isNullable)
                     || reader.TryReadWithConverter(nameof(InputDictionaryType.KeyType), options, ref keyType)
                     || reader.TryReadWithConverter(nameof(InputDictionaryType.ValueType), options, ref valueType);
 
@@ -43,7 +45,7 @@ public static InputDictionaryType CreateDictionaryType(ref Utf8JsonReader reader
             keyType = keyType ?? throw new JsonException("Dictionary must have key type");
             valueType = valueType ?? throw new JsonException("Dictionary must have value type");
 
-            return new InputDictionaryType(name ?? "Dictionary", keyType, valueType);
+            return new InputDictionaryType(name ?? "Dictionary", keyType, valueType, isNullable);
         }
     }
 }

src/AutoRest.CSharp/Common/Input/InputTypes/Serialization/TypeSpecInputEnumTypeConverter.cs

@@ -26,6 +26,7 @@ public override void Write(Utf8JsonWriter writer, InputEnumType value, JsonSeria
         public static InputEnumType CreateEnumType(ref Utf8JsonReader reader, string? id, string? name, JsonSerializerOptions options, ReferenceResolver resolver)
         {
             var isFirstProperty = id == null && name == null;
+            bool isNullable = false;
             string? ns = null;
             string? accessibility = null;
             string? deprecated = null;
@@ -39,6 +40,7 @@ public static InputEnumType CreateEnumType(ref Utf8JsonReader reader, string? id
             {
                 var isKnownProperty = reader.TryReadReferenceId(ref isFirstProperty, ref id)
                     || reader.TryReadString(nameof(InputEnumType.Name), ref name)
+                    || reader.TryReadBoolean(nameof(InputEnumType.IsNullable), ref isNullable)
                     || reader.TryReadString(nameof(InputEnumType.Namespace), ref ns)
                     || reader.TryReadString(nameof(InputEnumType.Accessibility), ref accessibility)
                     || reader.TryReadString(nameof(InputEnumType.Deprecated), ref deprecated)
@@ -98,7 +100,7 @@ public static InputEnumType CreateEnumType(ref Utf8JsonReader reader, string? id
             }
             valueType = currentType ?? throw new JsonException("Enum value type must be set.");
 
-            var enumType = new InputEnumType(name, ns, accessibility, deprecated, description, usage, valueType, NormalizeValues(allowedValues, valueType), isExtendable);
+            var enumType = new InputEnumType(name, ns, accessibility, deprecated, description, usage, valueType, NormalizeValues(allowedValues, valueType), isExtendable, IsNullable: isNullable);
             if (id != null)
             {
                 resolver.AddReference(id, enumType);

src/AutoRest.CSharp/Common/Input/InputTypes/Serialization/TypeSpecInputListTypeConverter.cs

@@ -25,11 +25,13 @@ public override void Write(Utf8JsonWriter writer, InputListType value, JsonSeria
         public static InputListType CreateListType(ref Utf8JsonReader reader, string? id, string? name, JsonSerializerOptions options)
         {
             var isFirstProperty = id == null && name == null;
+            bool isNullable = false;
             InputType? elementType = null;
             while (reader.TokenType != JsonTokenType.EndObject)
             {
                 var isKnownProperty = reader.TryReadReferenceId(ref isFirstProperty, ref id)
                     || reader.TryReadString(nameof(InputListType.Name), ref name)
+                    || reader.TryReadBoolean(nameof(InputListType.IsNullable), ref isNullable)
                     || reader.TryReadWithConverter(nameof(InputListType.ElementType), options, ref elementType);
 
                 if (!isKnownProperty)
@@ -39,7 +41,7 @@ public static InputListType CreateListType(ref Utf8JsonReader reader, string? id
             }
 
             elementType = elementType ?? throw new JsonException("List must have element type");
-            return new InputListType(name ?? "List", elementType);
+            return new InputListType(name ?? "List", elementType, isNullable);
         }
     }
 }

src/AutoRest.CSharp/Common/Input/InputTypes/Serialization/TypeSpecInputLiteralTypeConverter.cs

@@ -26,13 +26,15 @@ public override void Write(Utf8JsonWriter writer, InputLiteralType value, JsonSe
         public static InputLiteralType CreateInputLiteralType(ref Utf8JsonReader reader, string? id, string? name, JsonSerializerOptions options, ReferenceResolver resolver)
         {
             var isFirstProperty = id == null && name == null;
+            bool isNullable = false;
             Object? value = null;
             InputType? type = null;
 
             while (reader.TokenType != JsonTokenType.EndObject)
             {
                 var isKnownProperty = reader.TryReadReferenceId(ref isFirstProperty, ref id)
                     || reader.TryReadString(nameof(InputListType.Name), ref name)
+                    || reader.TryReadBoolean(nameof(InputListType.IsNullable), ref isNullable)
                     || reader.TryReadWithConverter(nameof(InputLiteralType.LiteralValueType), options, ref type);
 
                 if (isKnownProperty)
@@ -56,7 +58,7 @@ public static InputLiteralType CreateInputLiteralType(ref Utf8JsonReader reader,
 
             value = value ?? throw new JsonException("InputConstant must have value");
 
-            var literalType = new InputLiteralType(name, type, value);
+            var literalType = new InputLiteralType(name, type, value, isNullable);
 
             if (id != null)
             {