Merge spark deployment of master and history server

docker/playground/Dockerfile

@@ -1,6 +1,11 @@
 # syntax=docker/dockerfile:1
 FROM ghcr.io/fxttr/spark:latest
 
+ARG USERNAME=spark
+ARG USER_UID=185
+
+USER root
+
 RUN apt update && apt upgrade -y && \
   apt install -y --no-install-recommends \
   python3 \
@@ -29,8 +34,10 @@ COPY conf/ipython/startup/README /root/.ipython/profile_default/startup
 COPY conf/notebook ${BIN_DIR}/notebook
 COPY conf/notebook ${BIN_DIR}/pyspark-notebook
 
-RUN chmod u+x ${BIN_DIR}/notebook && chmod u+x ${BIN_DIR}/pyspark-notebook
+RUN chown ${USERNAME} ${BIN_DIR}/notebook && chmod u+x ${BIN_DIR}/notebook
+RUN chown ${USERNAME} ${BIN_DIR}/pyspark-notebook && chmod u+x ${BIN_DIR}/pyspark-notebook
 
 EXPOSE 8888
 
+USER ${USER_UID}
 CMD ["notebook"]

k8s/clickhouse/base/secrets.yaml

@@ -4,8 +4,8 @@ metadata:
     name: clickhouse-operator-secret
 type: Opaque
 stringData:
-    username: ENC[AES256_GCM,data:8jCUAhnVdgxuKLnwKlrg3Xhstg==,iv:II1ngue3DKTRPk0it/Q/Kg9Js/gNzQNL2R57a4Zs7no=,tag:GoR5HHKbNAnUOjaom7dePQ==,type:str]
-    password: ENC[AES256_GCM,data:Flq2sW9Hvy5ScF9jtKKq8RhOpN9NPlOS3A==,iv:nufP1R/HbMZ30+AhIFSzFVtoZGA5i4xvgmb4p2B0xrk=,tag:rdmRbjBxdyRAUpQRmNXy0g==,type:str]
+    username: ENC[AES256_GCM,data:X0njqHUNdmV1jDoxHaa1OL3awA==,iv:6MtaR6p/B37sKk4N5CJ5tMGsK3N/q3DreU5gdglQsv4=,tag:16ujsWSTLP+SXNSDbeDjiw==,type:str]
+    password: ENC[AES256_GCM,data:ID8c1MTkbDfebof2y2U0c9oj/Wm/mw9p8Q==,iv:EaECKCAEiLJIK14Dx5tIC7ImXEBa4nyV8QGmTf6T7wk=,tag:/+7qOiGksmaG7M8WODjqNA==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -15,14 +15,14 @@ sops:
         - recipient: age1dwu9ccy3t86c4tgd4ufs4jkmv572jfflkvdejm5kvjf95r3qsgus847qgd
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBySnkvbUFhK25xdURYSHUv
-            UHZPb0dmbU9VQ083bEJYWngwUG91ZHFUYVdRCm13Y0xmU25Kb2JJOGQzWW9XSDlS
-            ak82Y0JKYWdHZzBHVmw2L2RQVW9MQnMKLS0tIEx5a3FPYzVGZ3BkeG5DWjRJU1ZN
-            MU9qOHFYZHFURzV2MkwzZldFY0pWN2sKspB2jhBXZDsQGwma902Z8jtjAiew4Ug8
-            Hc6ZPrBhHhiqWw4yRDYV2f9wx/YjtA8/vakkEn9SOImjJmIZ7/jlWg==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhMnBEKzIwSUtvMDkxSzdB
+            dkM0Z2x1UEE5RStGTDBjSEFjMEhFR0I3M0dFCmZVSzZnb1BxMk9CZk14UHVkclN2
+            czNpTk1kT0JreWRPSER4MTBJWEhEZE0KLS0tIFhPeFVXeU43QVF6b0h1ZUNpWXZy
+            MjBYd2ZZYkRqaXJHUmpWN3Q3M2pDcFkKexMvNLMyFLARYxO48Wq04WrIPP+tdp7k
+            cmGuNAngFm1E3gh4J7TgRtDbKjm+klAGklpaX2q7vSb7MleMYAkyng==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-12-14T21:02:24Z"
-    mac: ENC[AES256_GCM,data:psdLcNIZwVHkSIinyJZf4dhVc0n88iKMvL3neQ37k0OUNuolDjKuKgDnzrIFCaA+lKAj+d2IFWhPj0XhyANqrwP4Qb4CXr6ca8iT8IFsIVzDVaiHiijtOHcoY+H+kyA7TG4NA3d2EIon9HsRe9DnSqpnTRlZIu9e1TGP6agL60g=,iv:CLQDlaSyKSNuaLlIzlhpwakXQmDYUhqj8FBxCveoeTE=,tag:BhpVMCYUwDapGypaU/Fa6Q==,type:str]
+    lastmodified: "2024-12-14T22:01:43Z"
+    mac: ENC[AES256_GCM,data:Zj7SEBAvZ/4T2Iga6wGw1bugTYzcmxH+3GhFsropt7O+vtH27vy32aqY/Fa+6IkoPerZBxBh9PzPNQ8wkt7Wh8oBoTR1Cdc1J42YltFbncCrqU+UrOVDMfaBxOpSQ9o1RmEBOvjp3KFmnM9NjlWq165ORs7BYvfSOvWmsCf29Ho=,iv:ED7/+YXkhxr//OwjnGs75VIu+oTujFXwGFTqVxzwUic=,tag:RheBnT30efJw+MqTNtYiSw==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.1
@@ -33,7 +33,7 @@ metadata:
     name: clickhouse-user-secret
 type: Opaque
 data:
-    password: ENC[AES256_GCM,data:tdoFFmfpIhFj0WPZdXyZAA==,iv:quVnu3XQJrgAXLcg4Hi0KtYKy/3cFMQ3DdnceTgNcIA=,tag:C8UKmCDUC3VHmtw4etTvOg==,type:str]
+    password: ENC[AES256_GCM,data:6WNJIxoSr7KT2ZUVg+RuIg==,iv:n6dj1LrNrhz/9L1YeDtbi0FD+nUEezzjgGJHRA0qOqk=,tag:2n2/o1lNluGq0J5BzROCUQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -43,14 +43,14 @@ sops:
         - recipient: age1dwu9ccy3t86c4tgd4ufs4jkmv572jfflkvdejm5kvjf95r3qsgus847qgd
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBySnkvbUFhK25xdURYSHUv
-            UHZPb0dmbU9VQ083bEJYWngwUG91ZHFUYVdRCm13Y0xmU25Kb2JJOGQzWW9XSDlS
-            ak82Y0JKYWdHZzBHVmw2L2RQVW9MQnMKLS0tIEx5a3FPYzVGZ3BkeG5DWjRJU1ZN
-            MU9qOHFYZHFURzV2MkwzZldFY0pWN2sKspB2jhBXZDsQGwma902Z8jtjAiew4Ug8
-            Hc6ZPrBhHhiqWw4yRDYV2f9wx/YjtA8/vakkEn9SOImjJmIZ7/jlWg==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhMnBEKzIwSUtvMDkxSzdB
+            dkM0Z2x1UEE5RStGTDBjSEFjMEhFR0I3M0dFCmZVSzZnb1BxMk9CZk14UHVkclN2
+            czNpTk1kT0JreWRPSER4MTBJWEhEZE0KLS0tIFhPeFVXeU43QVF6b0h1ZUNpWXZy
+            MjBYd2ZZYkRqaXJHUmpWN3Q3M2pDcFkKexMvNLMyFLARYxO48Wq04WrIPP+tdp7k
+            cmGuNAngFm1E3gh4J7TgRtDbKjm+klAGklpaX2q7vSb7MleMYAkyng==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-12-14T21:02:24Z"
-    mac: ENC[AES256_GCM,data:psdLcNIZwVHkSIinyJZf4dhVc0n88iKMvL3neQ37k0OUNuolDjKuKgDnzrIFCaA+lKAj+d2IFWhPj0XhyANqrwP4Qb4CXr6ca8iT8IFsIVzDVaiHiijtOHcoY+H+kyA7TG4NA3d2EIon9HsRe9DnSqpnTRlZIu9e1TGP6agL60g=,iv:CLQDlaSyKSNuaLlIzlhpwakXQmDYUhqj8FBxCveoeTE=,tag:BhpVMCYUwDapGypaU/Fa6Q==,type:str]
+    lastmodified: "2024-12-14T22:01:43Z"
+    mac: ENC[AES256_GCM,data:Zj7SEBAvZ/4T2Iga6wGw1bugTYzcmxH+3GhFsropt7O+vtH27vy32aqY/Fa+6IkoPerZBxBh9PzPNQ8wkt7Wh8oBoTR1Cdc1J42YltFbncCrqU+UrOVDMfaBxOpSQ9o1RmEBOvjp3KFmnM9NjlWq165ORs7BYvfSOvWmsCf29Ho=,iv:ED7/+YXkhxr//OwjnGs75VIu+oTujFXwGFTqVxzwUic=,tag:RheBnT30efJw+MqTNtYiSw==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.1

k8s/dagster/base/secrets.yaml

@@ -6,7 +6,7 @@ metadata:
         app: dagster-database
 type: Opaque
 data:
-    postgresql-password: ENC[AES256_GCM,data:5d+Nso/M+17NUB/dCWS1XpGZjgFRGz1X,iv:mU/gg3F62LXhsMimpuRQDDgJ2vj7Jc1VIeQH1+Q5aYY=,tag:sDRqGDnScdxvHXcwOeoc8Q==,type:str]
+    postgresql-password: ENC[AES256_GCM,data:KvQ+HpwvZECXqpOgqwal+VC1rH2mG1RL,iv:SYeveUfQSMZKYUBppQAmqYdJt8bP79eXi0cm0stTUo8=,tag:l5kAmVL0kyHEHxZ5jaWgjg==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -16,14 +16,14 @@ sops:
         - recipient: age1dwu9ccy3t86c4tgd4ufs4jkmv572jfflkvdejm5kvjf95r3qsgus847qgd
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJSDY2ZHEzMGYweGRiTUZh
-            SkZXSHJLUWxZUUh5SFRKV3hpQ0ExNUpvdVNJCm9KeFVhV3p5SmZGMU1kcmZTN1do
-            dHdJVUZpTWw0WDhjVVZUU1B2TnFtRkkKLS0tIE96QXNlOTBwU01PMTVjOGszc0Nv
-            cXBvRWg4VVFYZnVGbEt1TGFxZ0ZZVEEKxY4H/xrk4O3T6+1tc79VKEL6AV98eYe3
-            7AM7kx1INxjG71W9ZGNXqlcM+mNr+EE8EjEVcXgAtZdl6p2RR3afBQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBRDlDTjBFbGhMQ2QzTzFa
+            N1BhcjUzTVFxbFUxU2lZOFllM2h3S2laeWdzCjRBTzhWdjBzK2p5QkNIdWQwK2N6
+            SVlveGpzSXFiVGliMDhhQURqVllKWHcKLS0tIGtFTTVuRnI3ZXZka3J3Y3dvZ3FL
+            SWFhc3kwWXZlOWxOQmtuRTFBaTlHVXcKRJJyap7Osgh+jv+yp+Y0vroKNVEuS8Mo
+            SVWYbfytqaGD+Rsj8xOv2U9R+QP3c+tjauY6V0OwzVVVeN0or/QT7w==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-12-14T21:02:25Z"
-    mac: ENC[AES256_GCM,data:sOOuanYMLjoOjbO6ff1odRCv4PwaKepd1ilvnQex6AnsxWhSlf8g9u29yW6hZ7NNO9J62CQ+tMotS05oQcXS64TJaCe1aRsluY0IGrRRQcRoTS8tgGppi17gL+hLbO+Oe4VzxJRvwZYEu41KSIsO5gKfZCx0Fnggo/Dyz2Fgr3c=,iv:alBW8LJf6WTJcXy6LIMENXIk8DuZLym+JrOlHXqqjQ0=,tag:8+6deznjAc2lp2Voh+NUpA==,type:str]
+    lastmodified: "2024-12-14T22:01:43Z"
+    mac: ENC[AES256_GCM,data:lj8A62GqFpBoXcGg4jJM53Epg7H8B6TF/3jpjIQTjG76ZXBq2R0cO42HWxyBJhrVyVmHIhamxeGP2nbJhOdxaUbfiz3uYFtdWTlMYuWu/ucWVR6Rzjj49zorQv4Xfg1vjOvs6QwmEqIaZchWWcRynoW2rCKKAWDr1aQfSicXi/I=,iv:DkdLnjd/oEO3GQnTa2xPRlqTGrxXJsinW+VYRbdsGUg=,tag:TWI60+wgmsfCSG6U/NlcUw==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.1

k8s/minio/base/secrets.yaml

@@ -4,7 +4,7 @@ metadata:
     name: storage-configuration
 type: Opaque
 stringData:
-    config.env: ENC[AES256_GCM,data:jc7YoExX/tx4mdLslTJONh53VS9kNAfB+J2i72PhUXTCHCPgTTtmPs3mqT5OnE5HBURyXUg31pxoq/KH/nL/Dt2DyIhN7qtOhcC25XkdkUASiI1ObOko/qmlAH9bYPkmoo6Ch/i9OskiUnOw0wKAVCw3Uttzbg9/sp9qjKa1ktSsvvVYLf+Y1L/WxQQR,iv:yS4stIbr+kSqCgQyhwrlIGBVQHH68EPJEMyKPNvoEms=,tag:xuEscpItwrAQ2WTohm0m4g==,type:str]
+    config.env: ENC[AES256_GCM,data:v4jpzsTRhlJoegg7fevyEs41alM1/gRnrX9ZvpuQypuIpzmtKKaRDuFSojkA71gxQcjb8/m+fZ3emjgC4noh2EjW8hgCEjRFVnX0OM7LFoAOl2kgNB1o8IkQWQiLClbljr5IqcByf+iDxtiZRFOEPL/lkRTYN1GVQzPH/25+KhA3sNdQcO2/EWjA4MZc,iv:sbVqB14HVZgpppobmENQYR5+y0G48dyxKpHmt/tlKsM=,tag:dFDJgWzwE8vOEbjeCJkRwQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -14,14 +14,14 @@ sops:
         - recipient: age1dwu9ccy3t86c4tgd4ufs4jkmv572jfflkvdejm5kvjf95r3qsgus847qgd
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKY28yY2l2TmJxZUNIMlJy
-            bzkyOFdlTDdIZ0lrRDBnRjBkL2lCaWFBNkVVCnRUYm8ySE4rM3hWNExpS3Y1TVdh
-            aC8zN3ZZT3Rac2t3R0VDMzZLT0QyZDQKLS0tIEV3Q2dhTmhSUkJxaytrdW1TWlA3
-            TXR5SmhOV1lCOHNnV21pR3d4MnB1NlkKZbhrP7nfIAOt5BFZnx2SJvYvegqKqkMK
-            r4zNkacSIzKxFXqzd1yS6hhvqtZ/FU7I5eR6MZ6EVhNb5iMei1XXGQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0aUtXdHMzME5qTWlOM1NR
+            Njhlb0tFK0doZElTNVdaR0Z2N3NWbGJrem5BCkE3QTJHU1k4MFpiSlhYYUNqd1JQ
+            bEtYRFluV0thdHVFcUtVa0pmYTlqb2MKLS0tIFRNckhGUzU3YVRMZ1FndXppTWRZ
+            bDZCWm5KUVNYQ1c2K25JUnkyOUdxMkUK8YsVu+A50Y33ONDCzO5iRodQcQNMY2e4
+            JJ0gIhLIejJs/Jb739/h/8uOonC5IwUgOVF7YZKXXkyTNtJ6eAFrcA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-12-14T21:02:25Z"
-    mac: ENC[AES256_GCM,data:p+DRG3i55HKL+M0yyWB2F8rjjJfIk6RZ4A2YuYBTTSoMTwPxeHK51NOHgnAocKuD769hpIiRugfpc5j2C8aaMhNMBK6i3liAnrEyp99rsyXSN3YuQ32u6QMwkdC1yLRjphTST5/HRgc5Pkz5qWwSI55bMf8+ZsJdDo4FIJ5O+RY=,iv:I4bJrkmGgHLL8imjVT+AUYqc0lE/FGhx9OwTPAWUE6c=,tag:OjpF9+m5HA1gcLtPIaaG7w==,type:str]
+    lastmodified: "2024-12-14T22:01:43Z"
+    mac: ENC[AES256_GCM,data:Ovcl1g1UTnJJ3toIDDjTtcpVyIFp7XhcL72xsZWBWpNFaGIpa+7gZ7CHn3NsqPYmaT9ZXcm8BY6Eg6Mk4SKYX9puvrB5swlvEUuCANIkUqM8nPDGt3WQrs0BKTKn8i1ZTbaG21ZbyA2xKj/JspRZo9SAK7IXWdrB+4ZhXuaVlbE=,iv:ZaoZNLQjS/sdtGiwP3s+CAlFUedhTZ33K7roR3vcSrA=,tag:5jLMMEMSk6Fgok+irAnJVQ==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.1
@@ -32,8 +32,8 @@ metadata:
     name: storage-user
 type: Opaque
 data:
-    CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:sWVN6j4pYwIfsZ7z,iv:SSa36EN2HlUE5chPKloJ4tat8x+wIinCXF6kADRmgVo=,tag:n6JPNDIwxNEtZFz24Rq/dQ==,type:str]
-    CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:PlssLY+f4jUQZsrZ5TIgmA==,iv:dv+EDkElpmUhx1EZv4faMrt72TsTmHW2v8OWf7upMBQ=,tag:GfwppQBEf1BH9yxD8NZaEg==,type:str]
+    CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:mqG+6jN+WIlTBPlS,iv:er3c5nIAKP3srOm+nP+CcDseHb5yzrMo/9Ia6Wt+UbQ=,tag:rpWi9sEs5B8nxUOhJmnV3A==,type:str]
+    CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:JG+1z/VorEc8PTDqSLFs5Q==,iv:y6nMi5tju8MLQSos9Zn93N0LMa7fYaBEvXGJivp2N4I=,tag:qOyTF5ayOuFhNNHuKmvhvA==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -43,14 +43,14 @@ sops:
         - recipient: age1dwu9ccy3t86c4tgd4ufs4jkmv572jfflkvdejm5kvjf95r3qsgus847qgd
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKY28yY2l2TmJxZUNIMlJy
-            bzkyOFdlTDdIZ0lrRDBnRjBkL2lCaWFBNkVVCnRUYm8ySE4rM3hWNExpS3Y1TVdh
-            aC8zN3ZZT3Rac2t3R0VDMzZLT0QyZDQKLS0tIEV3Q2dhTmhSUkJxaytrdW1TWlA3
-            TXR5SmhOV1lCOHNnV21pR3d4MnB1NlkKZbhrP7nfIAOt5BFZnx2SJvYvegqKqkMK
-            r4zNkacSIzKxFXqzd1yS6hhvqtZ/FU7I5eR6MZ6EVhNb5iMei1XXGQ==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0aUtXdHMzME5qTWlOM1NR
+            Njhlb0tFK0doZElTNVdaR0Z2N3NWbGJrem5BCkE3QTJHU1k4MFpiSlhYYUNqd1JQ
+            bEtYRFluV0thdHVFcUtVa0pmYTlqb2MKLS0tIFRNckhGUzU3YVRMZ1FndXppTWRZ
+            bDZCWm5KUVNYQ1c2K25JUnkyOUdxMkUK8YsVu+A50Y33ONDCzO5iRodQcQNMY2e4
+            JJ0gIhLIejJs/Jb739/h/8uOonC5IwUgOVF7YZKXXkyTNtJ6eAFrcA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-12-14T21:02:25Z"
-    mac: ENC[AES256_GCM,data:p+DRG3i55HKL+M0yyWB2F8rjjJfIk6RZ4A2YuYBTTSoMTwPxeHK51NOHgnAocKuD769hpIiRugfpc5j2C8aaMhNMBK6i3liAnrEyp99rsyXSN3YuQ32u6QMwkdC1yLRjphTST5/HRgc5Pkz5qWwSI55bMf8+ZsJdDo4FIJ5O+RY=,iv:I4bJrkmGgHLL8imjVT+AUYqc0lE/FGhx9OwTPAWUE6c=,tag:OjpF9+m5HA1gcLtPIaaG7w==,type:str]
+    lastmodified: "2024-12-14T22:01:43Z"
+    mac: ENC[AES256_GCM,data:Ovcl1g1UTnJJ3toIDDjTtcpVyIFp7XhcL72xsZWBWpNFaGIpa+7gZ7CHn3NsqPYmaT9ZXcm8BY6Eg6Mk4SKYX9puvrB5swlvEUuCANIkUqM8nPDGt3WQrs0BKTKn8i1ZTbaG21ZbyA2xKj/JspRZo9SAK7IXWdrB+4ZhXuaVlbE=,iv:ZaoZNLQjS/sdtGiwP3s+CAlFUedhTZ33K7roR3vcSrA=,tag:5jLMMEMSk6Fgok+irAnJVQ==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.1

k8s/nessie/base/secrets.yaml

@@ -6,9 +6,9 @@ metadata:
         app: nessie-database
 type: Opaque
 data:
-    db: ENC[AES256_GCM,data:fJYr5gT8l/4=,iv:0pxQ1ARQHQoW2WsZkqmH4jgobgCypg01bRsMMctTNoA=,tag:q5WAB3FDD6gzA6cKHl5BnQ==,type:str]
-    username: ENC[AES256_GCM,data:ds+mPO7eE3Y=,iv:yD1FfYg1J1hgtSfVYMdcTfgUK/mXHlQyCWOGWSq1d68=,tag:YcF4gAfJDyXEZrPQ/fMqWQ==,type:str]
-    password: ENC[AES256_GCM,data:Hmidu+M1x+KCnTfeb1IsQg==,iv:+i1puk7HMK6ZxXNS8g1K9iPanUabRz9fvy/aW1m2KQ0=,tag:khv6SUIIaDWzihdreGakhA==,type:str]
+    db: ENC[AES256_GCM,data:XxnMEgqG/yo=,iv:nvhPS88rwT6dHQqpqojs/WgNz6F/xvB7QIuJUndHRvU=,tag:PkNGPcuvzhKT01/bBH6kzA==,type:str]
+    username: ENC[AES256_GCM,data:Z052Nq/Hipo=,iv:QoD7+ZvzNf4hNRrN+CweA+fupVJvfc0/5p81QzSuKSY=,tag:Ji+tPjH78JnYKPyPL+9WIA==,type:str]
+    password: ENC[AES256_GCM,data:P058yJLklg5vzUUXL1Sq1g==,iv:LV5Yq4EZ4cnF8HkZTRN2cKn2j2u/KkqYue1n0kXC7hQ=,tag:J8kqkW1wxNO09XazOfO/pA==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -18,14 +18,14 @@ sops:
         - recipient: age1dwu9ccy3t86c4tgd4ufs4jkmv572jfflkvdejm5kvjf95r3qsgus847qgd
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5ejJuOHZ4aE9zc0lNYWJ1
-            TmZWVnJ5QzJEdHIwZmFmd1AvM3ZIY0tvZmcwCnl1NFFITUJCL2E0NlRLYnRqeUV1
-            QUh6cVNJVzNDbStRS3ozWnhmcmdxUDgKLS0tIDhFOEMwWitxWDNLUDZxZ05QZEFF
-            RlVva0FTcUFMYUlaUkJRd3BIK241aFUK6p9nf0OJTPjXnMegtHfYaJJhrKryZ3In
-            LWq4PWcrP/VA3TmogeBs/5w/zAXHCeZrkIsfqiOC7VNu+nvMP5L2Xw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKRmp4NUthY0R2L0FIT2hU
+            cHBWL085KyttOU92ZEozQjV5anFDZ000V3g0CldXcjlieGpFcnRMVTVNbmZJbnJu
+            eU1yOFNYZy9SdG9uZnVaMVRzeUN3Q0kKLS0tIHV4SU14SklVV3M5NjhtVWgwcGlj
+            bnJGakd0dEw5QmlWSUkvalFqSU0rMnMKZED/lMQMIPlBZfcr8neLgiD69QP92JYl
+            D2BDmiaJ6sbB1AWsyJ9Z048ct/+EfVLaO3cy8s9TZBVbg0wK3NTfpg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-12-14T21:02:25Z"
-    mac: ENC[AES256_GCM,data:+JRiWREXdL69uWP9iittTQENPWKSEyFy3dou2oRKbSE3AzP8syursf9QrOdqV3BryRSoXPf5jw2nAiZT13LvcUY4mUBsPKxF2Fo/r0aO3aoadJEwEOGQHp87P73b7teHnAggkzuPC77SiTL3OaXqfXFQ42rkjLfVoeSQU+VupvA=,iv:fRCcnsuT6PBTLD8rcTJZFGwirYxmO/UedB3zS9+atFI=,tag:9NV6TGYTq5D6PSIubZocaQ==,type:str]
+    lastmodified: "2024-12-14T22:01:43Z"
+    mac: ENC[AES256_GCM,data:XoDzfvXQvTRfAkfu9EOMunFOm8pKeBnWf1RgLILG68O1h8b/GoZwHISgMHGa/q5Sq9uRp074h9juuCmk6wCC236QSY7lfq7ijwBVFxqElq3fxM4/N3DCiycrfS0hxBQnoZbmmUwf4WBFdfp8pGlvaOR0no00EB0e0y7rmLZkcgs=,iv:3LLQMKxR4glCT+moS5272JDRCWVCEq0MdJKZLkwuNo4=,tag:WBCNirCz3beOImhZ34T5gg==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.1

k8s/spark/base/kustomization.yaml

@@ -9,6 +9,4 @@ resources:
   - master-service.yaml
   - worker-deployment.yaml
   - worker-service.yaml
-  - history-server-deployment.yaml
-  - history-server-service.yaml
   - secrets.yaml

k8s/spark/base/master-deployment.yaml

@@ -20,6 +20,12 @@ spec:
       serviceAccountName: spark
       imagePullSecrets:
         - name: dockerconfigjson
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 185
+        runAsGroup: 185
+        seccompProfile:
+          type: "RuntimeDefault"
       containers:
         - name: spark-master
           image: ghcr.io/fxttr/spark:latest
@@ -54,6 +60,41 @@ spec:
           volumeMounts:
             - mountPath: /opt/spark/logs
               name: spark-logs
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+        - name: spark-history-server
+          image: ghcr.io/fxttr/spark:latest
+          imagePullPolicy: Always
+          command: 
+            - "start-history-server.sh"
+          ports:
+            - containerPort: 18080
+          env:
+            - name: SPARK_MODE
+              value: "history-server"
+            - name: SPARK_NO_DAEMONIZE
+              value: "true"
+            - name: SPARK_HISTORY_OPTS
+              value: "-Dspark.history.fs.logDirectory=/opt/spark/logs -Dspark.history.ui.port=18080"
+          envFrom:
+            - secretRef:
+                name: spark-env
+          resources:
+            requests:
+              memory: "1Gi"
+              cpu: "1"
+            limits:
+              memory: "4Gi"
+              cpu: "2"
+          volumeMounts:
+            - mountPath: /opt/spark/logs
+              name: spark-logs
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
       volumes:
         - name: spark-logs
           emptyDir: {}

k8s/spark/base/master-service.yaml

@@ -10,7 +10,8 @@ spec:
       name: spark-master
     - port: 8080
       name: spark-ui
+    - port: 18080
+      name: history-server-ui
   selector:
     app: spark
     component: master
-  clusterIP: None # Headless service for easier resolution