[SPARK-43341][SQL] Patch StructType.toDDL not picking up on non-nullability of nested column

sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala

@@ -154,14 +154,22 @@ case class StructField(
    */
   private[sql] def sql = s"${QuotingUtils.quoteIfNeeded(name)}: ${dataType.sql}$getDDLComment"
 
+  private[sql] def toDDL(isNested: Boolean): String = {
+    val nullString = if (nullable) "" else " NOT NULL"
+    val dataTypeDDL = dataType match {
+      case s: StructType => s.toNestedDDL
+      case d: DataType => d.sql
+    }
+    val separator = if (isNested) ": " else " "
+    val quotedName = QuotingUtils.quoteIfNeeded(name)
+    s"$quotedName$separator$dataTypeDDL$nullString$getDDLDefault$getDDLComment"
+  }
+
   /**
    * Returns a string containing a schema in DDL format. For example, the following value:
    * `StructField("eventId", IntegerType, false)` will be converted to `eventId` INT NOT NULL.
    * `StructField("eventId", IntegerType, true)` will be converted to `eventId` INT.
    * @since 2.4.0
    */
-  def toDDL: String = {
-    val nullString = if (nullable) "" else " NOT NULL"
-    s"${QuotingUtils.quoteIfNeeded(name)} ${dataType.sql}${nullString}$getDDLDefault$getDDLComment"
-  }
+  def toDDL: String = toDDL(isNested = false)
 }

sql/api/src/main/scala/org/apache/spark/sql/types/StructType.scala

@@ -450,6 +450,9 @@ case class StructType(fields: Array[StructField]) extends DataType with Seq[Stru
    */
   def toDDL: String = fields.map(_.toDDL).mkString(",")
 
+  private[sql] def toNestedDDL: String =
+    s"STRUCT<${fields.map(_.toDDL(isNested = true)).mkString(", ")}>"
+
   private[sql] override def simpleString(maxNumberFields: Int): String = {
     val builder = new StringBuilder
     val fieldTypes = fields.take(maxNumberFields).map {

sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala

@@ -86,19 +86,19 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
   private val nestedStruct = new StructType()
     .add(StructField("a", new StructType()
       .add(StructField("b", new StructType()
-        .add(StructField("c", StringType
+        .add(StructField("c", StringType, false
         ).withComment("Deep Nested comment"))
       ).withComment("Nested comment"))
     ).withComment("comment"))
 
   test("SPARK-33846: toDDL should output nested field's comment") {
-    val ddl = "a STRUCT<b: STRUCT<c: STRING COMMENT 'Deep Nested comment'> " +
+    val ddl = "a STRUCT<b: STRUCT<c: STRING NOT NULL COMMENT 'Deep Nested comment'> " +
       "COMMENT 'Nested comment'> COMMENT 'comment'"
     assert(nestedStruct.toDDL == ddl)
   }
 
   test("SPARK-33846: fromDDL should parse nested field's comment") {
-    val ddl = "`a` STRUCT<`b`: STRUCT<`c`: STRING COMMENT 'Deep Nested comment'> " +
+    val ddl = "`a` STRUCT<`b`: STRUCT<`c`: STRING NOT NULL COMMENT 'Deep Nested comment'> " +
       "COMMENT 'Nested comment'> COMMENT 'comment'"
     assert(StructType.fromDDL(ddl) == nestedStruct)
   }
@@ -108,7 +108,7 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
   }
 
   test("SPARK-35706: make the ':' in STRUCT data type definition optional") {
-    val ddl = "`a` STRUCT<`b` STRUCT<`c` STRING COMMENT 'Deep Nested comment'> " +
+    val ddl = "`a` STRUCT<`b` STRUCT<`c` STRING NOT NULL COMMENT 'Deep Nested comment'> " +
       "COMMENT 'Nested comment'> COMMENT 'comment'"
     assert(StructType.fromDDL(ddl) == nestedStruct)
   }