omitempty corrections

api/v1beta2/scheduledsparkapplication_types.go

@@ -88,9 +88,9 @@ type ScheduledSparkApplicationStatus struct {
 // ScheduledSparkApplication is the Schema for the scheduledsparkapplications API.
 type ScheduledSparkApplication struct {
 	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
+	metav1.ObjectMeta `json:"metadata"`
 
-	Spec   ScheduledSparkApplicationSpec   `json:"spec,omitempty"`
+	Spec   ScheduledSparkApplicationSpec   `json:"spec"`
 	Status ScheduledSparkApplicationStatus `json:"status,omitempty"`
 }
 

api/v1beta2/sparkapplication_types.go

@@ -62,7 +62,6 @@ type SparkApplicationSpec struct {
 	// +optional
 	MainClass *string `json:"mainClass,omitempty"`
 	// MainFile is the path to a bundled JAR, Python, or R file of the application.
-	// +optional
 	MainApplicationFile *string `json:"mainApplicationFile"`
 	// Arguments is a list of arguments to be passed to the application.
 	// +optional
@@ -187,9 +186,9 @@ type SparkApplicationStatus struct {
 // SparkApplication is the Schema for the sparkapplications API
 type SparkApplication struct {
 	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
+	metav1.ObjectMeta `json:"metadata"`
 
-	Spec   SparkApplicationSpec   `json:"spec,omitempty"`
+	Spec   SparkApplicationSpec   `json:"spec"`
 	Status SparkApplicationStatus `json:"status,omitempty"`
 }
 
@@ -280,15 +279,15 @@ type SparkUIConfiguration struct {
 	// ServicePort allows configuring the port at service level that might be different from the targetPort.
 	// TargetPort should be the same as the one defined in spark.ui.port
 	// +optional
-	ServicePort *int32 `json:"servicePort"`
+	ServicePort *int32 `json:"servicePort,omitempty"`
 	// ServicePortName allows configuring the name of the service port.
 	// This may be useful for sidecar proxies like Envoy injected by Istio which require specific ports names to treat traffic as proper HTTP.
 	// Defaults to spark-driver-ui-port.
 	// +optional
-	ServicePortName *string `json:"servicePortName"`
+	ServicePortName *string `json:"servicePortName,omitempty"`
 	// ServiceType allows configuring the type of the service. Defaults to ClusterIP.
 	// +optional
-	ServiceType *corev1.ServiceType `json:"serviceType"`
+	ServiceType *corev1.ServiceType `json:"serviceType,omitempty"`
 	// ServiceAnnotations is a map of key,value pairs of annotations that might be added to the service object.
 	// +optional
 	ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"`
@@ -312,7 +311,7 @@ type DriverIngressConfiguration struct {
 	ServicePortName *string `json:"servicePortName"`
 	// ServiceType allows configuring the type of the service. Defaults to ClusterIP.
 	// +optional
-	ServiceType *corev1.ServiceType `json:"serviceType"`
+	ServiceType *corev1.ServiceType `json:"serviceType,omitempty"`
 	// ServiceAnnotations is a map of key,value pairs of annotations that might be added to the service object.
 	// +optional
 	ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"`

charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -11573,6 +11573,7 @@ spec:
                 required:
                 - driver
                 - executor
+                - mainApplicationFile
                 - sparkVersion
                 - type
                 type: object
@@ -11621,6 +11622,9 @@ spec:
                   application.
                 type: string
             type: object
+        required:
+        - metadata
+        - spec
         type: object
     served: true
     storage: true

charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml

@@ -11475,6 +11475,7 @@ spec:
             required:
             - driver
             - executor
+            - mainApplicationFile
             - sparkVersion
             - type
             type: object
@@ -11555,6 +11556,9 @@ spec:
             required:
             - driverInfo
             type: object
+        required:
+        - metadata
+        - spec
         type: object
     served: true
     storage: true

config/crd/bases/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -11573,6 +11573,7 @@ spec:
                 required:
                 - driver
                 - executor
+                - mainApplicationFile
                 - sparkVersion
                 - type
                 type: object
@@ -11621,6 +11622,9 @@ spec:
                   application.
                 type: string
             type: object
+        required:
+        - metadata
+        - spec
         type: object
     served: true
     storage: true

config/crd/bases/sparkoperator.k8s.io_sparkapplications.yaml

@@ -11475,6 +11475,7 @@ spec:
             required:
             - driver
             - executor
+            - mainApplicationFile
             - sparkVersion
             - type
             type: object
@@ -11555,6 +11556,9 @@ spec:
             required:
             - driverInfo
             type: object
+        required:
+        - metadata
+        - spec
         type: object
     served: true
     storage: true

docs/api-docs.md

@@ -1913,7 +1913,6 @@ string
 </em>
 </td>
 <td>
-<em>(Optional)</em>
 <p>MainFile is the path to a bundled JAR, Python, or R file of the application.</p>
 </td>
 </tr>
@@ -2355,7 +2354,6 @@ string
 </em>
 </td>
 <td>
-<em>(Optional)</em>
 <p>MainFile is the path to a bundled JAR, Python, or R file of the application.</p>
 </td>
 </tr>

internal/controller/scheduledsparkapplication/controller_test.go

@@ -29,6 +29,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/kubeflow/spark-operator/api/v1beta2"
+	"github.com/kubeflow/spark-operator/pkg/util"
 )
 
 var _ = Describe("ScheduledSparkApplication Controller", func() {
@@ -61,6 +62,7 @@ var _ = Describe("ScheduledSparkApplication Controller", func() {
 							RestartPolicy: v1beta2.RestartPolicy{
 								Type: v1beta2.RestartPolicyNever,
 							},
+							MainApplicationFile: util.StringPtr("local:///dummy.jar"),
 						},
 					},
 					// TODO(user): Specify other spec details if needed.

internal/controller/sparkapplication/controller_test.go

@@ -95,6 +95,9 @@ var _ = Describe("SparkApplication Controller", func() {
 							common.LabelSparkAppName: app.Name,
 						},
 					},
+					Spec: v1beta2.SparkApplicationSpec{
+						MainApplicationFile: util.StringPtr("local:///dummy.jar"),
+					},
 				}
 				v1beta2.SetSparkApplicationDefaults(app)
 				Expect(k8sClient.Create(ctx, app)).To(Succeed())
@@ -146,6 +149,9 @@ var _ = Describe("SparkApplication Controller", func() {
 						Name:      appName,
 						Namespace: appNamespace,
 					},
+					Spec: v1beta2.SparkApplicationSpec{
+						MainApplicationFile: util.StringPtr("local:///dummy.jar"),
+					},
 				}
 				v1beta2.SetSparkApplicationDefaults(app)
 				Expect(k8sClient.Create(ctx, app)).To(Succeed())
@@ -202,6 +208,9 @@ var _ = Describe("SparkApplication Controller", func() {
 						Name:      appName,
 						Namespace: appNamespace,
 					},
+					Spec: v1beta2.SparkApplicationSpec{
+						MainApplicationFile: util.StringPtr("local:///dummy.jar"),
+					},
 				}
 				v1beta2.SetSparkApplicationDefaults(app)
 				Expect(k8sClient.Create(ctx, app)).To(Succeed())
@@ -253,6 +262,9 @@ var _ = Describe("SparkApplication Controller", func() {
 						Name:      appName,
 						Namespace: appNamespace,
 					},
+					Spec: v1beta2.SparkApplicationSpec{
+						MainApplicationFile: util.StringPtr("local:///dummy.jar"),
+					},
 				}
 				v1beta2.SetSparkApplicationDefaults(app)
 				Expect(k8sClient.Create(ctx, app)).To(Succeed())
@@ -309,6 +321,9 @@ var _ = Describe("SparkApplication Controller", func() {
 						Name:      appName,
 						Namespace: appNamespace,
 					},
+					Spec: v1beta2.SparkApplicationSpec{
+						MainApplicationFile: util.StringPtr("local:///dummy.jar"),
+					},
 				}
 				v1beta2.SetSparkApplicationDefaults(app)
 				Expect(k8sClient.Create(ctx, app)).To(Succeed())