From 8de55516a363b2b49c11f5d165e809ef8c60842b Mon Sep 17 00:00:00 2001 From: Min RK Date: Wed, 18 Sep 2024 10:28:53 +0200 Subject: [PATCH 1/2] add kbatch to user image --- .../jupyterhub/images/user/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/gfts-track-reconstruction/jupyterhub/images/user/requirements.txt b/gfts-track-reconstruction/jupyterhub/images/user/requirements.txt index 62138f7..53b3d6d 100644 --- a/gfts-track-reconstruction/jupyterhub/images/user/requirements.txt +++ b/gfts-track-reconstruction/jupyterhub/images/user/requirements.txt @@ -6,6 +6,7 @@ gh-scoped-creds isort jupyter-keepalive jupyterhub==4.1.6 +kbatch @ git+https://github.com/kbatch-dev/kbatch#subdirectory=kbatch # git+https://github.com/iaocea/pangeo-fish#egg=pangeo-fish #git+https://github.com/iaocea/xarray-regridding#egg=xarray-regridding xarray-healpy @ git+https://github.com/iaocea/xarray-regridding.git@0ffca6058f4008f4f22f076e2d60787fcf32ac82 From da01a637e6be699fcfd74a78425ea6101943062a Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 20 Sep 2024 12:44:30 +0200 Subject: [PATCH 2/2] deploy kbatch --- .pre-commit-config.yaml | 2 +- .../jupyterhub/gfts-hub/Chart.lock | 7 +- .../jupyterhub/gfts-hub/Chart.yaml | 5 + .../gfts-hub/templates/kbatch-rbac.yaml | 31 +++++ .../jupyterhub/gfts-hub/values.yaml | 112 ++++++++++++++++++ .../jupyterhub/secrets/config.yaml | Bin 3395 -> 3621 bytes 6 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 gfts-track-reconstruction/jupyterhub/gfts-hub/templates/kbatch-rbac.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e67cf16..3aaee28 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,7 +38,7 @@ repos: hooks: - id: chartpress # run in subdirectory - entry: bash -c 'cd gfts-track-reconstruction/jupyterhub && "$@"' -- + entry: bash -c 'cd gfts-track-reconstruction/jupyterhub && chartpress --reset' # Autoformat and linting, misc. details - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.lock b/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.lock index 33b1b43..891fb2b 100644 --- a/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.lock +++ b/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.lock @@ -5,6 +5,9 @@ dependencies: - name: dask-gateway repository: https://helm.dask.org/ version: 2024.1.0 +- name: kbatch-proxy + repository: https://kbatch-dev.github.io/helm-chart + version: 0.4.2 - name: ingress-nginx repository: https://kubernetes.github.io/ingress-nginx version: 4.9.1 @@ -14,5 +17,5 @@ dependencies: - name: grafana repository: https://grafana.github.io/helm-charts version: 7.0.14 -digest: sha256:f65f939aed209d2ebec4ea4534deef0d14d8b99ee1c5ee20ddee11d0a7115b20 -generated: "2024-04-03T14:18:11.909303+02:00" +digest: sha256:537e964a33edb2d34d66718427bc2b0716efc7c299330414696b1e80eb3f91ac +generated: "2024-09-04T14:58:24.438886+02:00" diff --git a/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.yaml b/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.yaml index a515cc3..c8eb9a8 100644 --- a/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.yaml +++ b/gfts-track-reconstruction/jupyterhub/gfts-hub/Chart.yaml @@ -14,6 +14,11 @@ dependencies: version: "2024.1.0" repository: https://helm.dask.org/ + # kbatch + - name: kbatch-proxy + version: "0.4.2" + repository: https://kbatch-dev.github.io/helm-chart + # ingress - name: ingress-nginx version: "4.9.1" diff --git a/gfts-track-reconstruction/jupyterhub/gfts-hub/templates/kbatch-rbac.yaml b/gfts-track-reconstruction/jupyterhub/gfts-hub/templates/kbatch-rbac.yaml new file mode 100644 index 0000000..f264f23 --- /dev/null +++ b/gfts-track-reconstruction/jupyterhub/gfts-hub/templates/kbatch-rbac.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kbatch-role +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["create", "get", "list", "delete"] + + - apiGroups: [""] + resources: ["pods", "pods/log", "configmaps"] + verbs: ["get", "watch", "list", "create", "delete", "patch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kbatch +subjects: + - kind: User + name: system:serviceaccount:{{ .Release.Namespace }}:kbatch-proxy + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: kbatch-role + apiGroup: rbac.authorization.k8s.io diff --git a/gfts-track-reconstruction/jupyterhub/gfts-hub/values.yaml b/gfts-track-reconstruction/jupyterhub/gfts-hub/values.yaml index d1ee534..8d5550a 100644 --- a/gfts-track-reconstruction/jupyterhub/gfts-hub/values.yaml +++ b/gfts-track-reconstruction/jupyterhub/gfts-hub/values.yaml @@ -30,6 +30,16 @@ jupyterhub: name: "c63eqfuv.c1.gra9.container-registry.ovh.net/gfts/jupyterhub-user" tag: "set-by-chartpress" pullPolicy: Always + networkPolicy: + egress: + # allow access to kbatch-proxy + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: kbatch-proxy + ports: + - protocol: TCP + port: 80 memory: limit: 24G guarantee: 8G @@ -43,6 +53,7 @@ jupyterhub: CULL_INTERVAL: "120" GH_SCOPED_CREDS_CLIENT_ID: Iv1.f4a7db20c671f599 GH_SCOPED_CREDS_APP_URL: https://github.com/apps/gfts-jupyterhub + KBATCH_URL: http://kbatch-proxy AWS_ENDPOINT_URL_S3: "https://s3.gra.perf.cloud.ovh.net" AWS_DEFAULT_REGION: gra # JUPYTER_FS_BUCKETS: destine-gfts-data-lake,gfts-reference-data,gfts-ifremer @@ -186,6 +197,11 @@ jupyterhub: c.KubeSpawner.pre_spawn_hook = load_creds loadRoles: + server: + scopes: + - access:servers!server + - users:activity!user + - access:services!service=kbatch management: scopes: - admin-ui @@ -197,6 +213,13 @@ jupyterhub: users: - annefou - minrk + kbatch-users: + scopes: + - "access:services!service=kbatch" + users: + - minrk + services: + kbatch-proxy: {} scheduling: userScheduler: @@ -314,3 +337,92 @@ dask-gateway: # shutdown idle clusters after one hour c.ClusterConfig.idle_timeout = 3600 + +kbatch-proxy: + fullnameOverride: kbatch-proxy + image: + repository: ghcr.io/minrk/kbatch-proxy + tag: "gfts" + pullPolicy: Always + app: + # jupyterhub_service_prefix: /services/kbatch-proxy + # cannot use internal ip + # pending https://github.com/kbatch-dev/helm-chart/pull/6 + jupyterhub_api_url: https://gfts.minrk.net/hub/api/ + extra_env: + JUPYTERHUB_SERVICE_NAME: kbatch + KBATCH_JOB_TEMPLATE_FILE: /srv/job_template.yaml + KBATCH_PROFILE_FILE: /srv/profiles.yaml + extraFiles: + job_template: + mountPath: /srv/job_template.yaml + data: + apiVersion: batch/v1 + kind: Job + metadata: + labels: + app.kubernetes.io/managed-by: kbatch + spec: + template: + metadata: + labels: + app.kubernetes.io/managed-by: kbatch + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: hub.jupyter.org/node-purpose + operator: In + values: + - user + profiles: + mountPath: /srv/profiles.yaml + data: + default: + resources: + requests: + cpu: "2" + memory: "8G" + limits: + cpu: "2" + memory: "8G" + big60: + resources: + requests: + cpu: "7" + memory: "60G" + limits: + cpu: "8" + memory: "60G" + big120: + resources: + requests: + cpu: "14" + memory: "120G" + limits: + cpu: "15" + memory: "120G" + big240: + resources: + requests: + cpu: "29" + memory: "240G" + limits: + cpu: "30" + memory: "240G" + huge: + resources: + requests: + cpu: "59" + memory: "480G" + limits: + cpu: "60" + memory: "480G" + node_affinity_required: + - matchExpressions: + - key: gfts.destination-earth.eu/size + operator: In + values: + - big512 diff --git a/gfts-track-reconstruction/jupyterhub/secrets/config.yaml b/gfts-track-reconstruction/jupyterhub/secrets/config.yaml index 671b8bd66565c2986c9c90d57ee31ef85dbec76c..b04233e22819aa98519fb95d07d804153d945d14 100644 GIT binary patch literal 3621 zcmV+=4%+bmM@dveQdv+`0PEH=vGaRDO3BD%t|z1t=g5Hy`mw73><=(R;9yV`S;Bns zs?nWiRl`j)ecp^96RG^{fn^Y#fA|6!0WKV>x{`1jR-Stp-X4+PcB6 zwAZVIYfF!13VI4BIDb%iyAD?E+>^4%9E&c(%>+KsQJIM0#HJ51BD6SivFId3z*37H zg>a=EsXI#7U~*f;4;(@wL%I-whlUnS86Hz>lqbs|_jkv2RaU*&dM4lg$Z(|QqZsJ` zE(N?d#y$wG%)*!Ku;Woms$W>Z?M?$q_y*6(Qj`qmKu)o|)J_L%S2QNL=wp!DYV z%ZhvM_(jFG;0e=(TtM$Jct0DnXQ9@e7a?~GkM*B~W^vk;f!IR6N1nK7rbtH2b)I;w z0lY825G@1{!s&=FhoTh<3+W#dtC6x(!{ z*OpTcJNSN9n6}aZ7W7;bC1i@`n~e>gIfd%rsz?TB;(e6Zr|znG*a$sXbw-7-gkoGX@}F790n@Wo&ZIl zlT*(NAr21K@jvgbRdM>o1d`2TuvOGu%dDlqE+79UcU0yG#DL_M8xg3@UjTfbCeP+} z?%Lpzr9bGN{@pzu+>JlQflKeM=)jPj{2?ELKillYADJ-eZl}S8-A>9jncB2DFt^3;OeUEt*}& zj_M65l02dQQGS*yl-6uZao3sK2D9w~lxj5d$d7G>1&&w*S;iVYYw;SXkpggJH{?ef z4Q7`{dx8iO+=v?Kc@%=KfC1Gj1Af=o?`Q9L9l4eQDxGdaj#KcZ5<8=YwR6e*L5pIg zq*^2ElPB0eMnb&ecn1Pf%bbK(UA*Sb^Z>X;o@9F;Q&{lGhsKL9Ff29tf6CUyZV)=` z&!asN&PnB(VPAGqxC>aN>Chr#=k(wWh3g@It;8`MQW^upy=gUM^`U52Ig}%^Xq#nI z9*5%|gi(W*%vQgv0BztlwU=f}qUmH2AnXu9eaocM!`oBO9+9W_ko_7LS#_&@`He^! zI^u!ZXm)%YyOC#G!qip)Zs;JSaD}0ZblGZ}P_qmuz$kRA(;t~s8IB3$ucXQs!J83PL8$~xic-%!iUgEkY17N*KCFq_rK9>y`-L-CV#dwe^*XIz9J@B=ax z0(N>nhtC}`?C1Zf13bgf^T6rBq!hg5e;-f#1}_{sS3WY{r}TJ+LriHk>rWkZ(rZa^{e*c3a z*g4D4$Vu0C|GznZYMq}nC%ne6Nj$I{dnkX_?D%_)&3$BeH-g^G3O--iV}r>Z1~%F1 z&v2aA5IMk0`5z19{i;l7^^i}}e1}=#wR0gRKTyP+v8oaebaJWlRz0j+{|cp!qee6{ z=HD*r7~pgcXQoqD1FAU$jv{C-|F!tRBJ^uB_7;FE8povB(w&3t*Un_WdY3D@nx^nZ znM%aSJ(Q&c-a+`2=Mq%03OVx{QL1`b-{h_+d0~z6K1*S8 z+c-Su_^8iEN$ma1px&*x9ZXGfpgYJi_uW3CYVamhdJr7{ru3W)dmb-udvW6@FsBos zjUJB!CTD`8z*Um7xRQxs4Al6!$>B2!J1pLW5q=`?t9J;8y*iN z4`+*iWNO(gUgr~C8)Tc%vsBYPMCg=^?@)gm~(Y@+=>#_O<$@wK1IN<~XyDxd40swBl}m@}bV zBb&z_l~N3dBSPt+nXe91c$32S(Vib#(17QqCA<8!KJ_8r_cMT z6Gis%$#o`*#qj>ji8I?mt^$CPIK~ZzQhxxA6R5rft8RObsoZg8Pn!`Clyg2SR!vEd z3wyaSv`i1I+uX37Z526QKw5rDGS)?u=_ zhlQlO#m1}Z+o&IZzs-$SkDfKvF^3`cD_i%GZ7@N_^#CwZXXif>{r*{%8L@ff-_VX zR*4UfLP=V!VLt5+(mA?2}tH0U@hPW)&-CI=HO0X%It_z zl=2nHrT|+nRB7crzdFj)wQ8LbF&_k#gHYtqHOXY`6^XQj%|bd^CE67=c>tUg7fkn{ z6&_98+=ho9dicN9uoFOaJr!v;K@6x`NEtkQtb$iA$8mwhkQciR-T{8*-rM67e@?k* z<3!)U-kPKB?na%N;2CkynWwq&x6!YRTr{Pl1fG(6d^D>#Ltx%8ux@F%R5UEbB#O%$ zJ<~`1Nh-T%Q>{Df$QCX1xg>2L7k1mSiRjd9G2kQSKnoH~q8(j|5V0uPqY$D~K+!$T z2AP}K*YlL=)weO9qGO>1O$BoeUwA9C#=j|Rlyu9PuLQDqns6Aq`sW9sK^$9Mbd|oF z5p!abZu7m1J52trXV4q;>k>X-9`b8*4zJeb3XV@Uneiy1f1nMMxOC4X+D}O*<{1h2 zDEZl{Kx~W5bbKg~A|RmfLFj5HNfweY>Dk|(IZ#7;cNlBg(?sD*inZ0v8{IV|Nb=>U zoH&)@vqV4o`|lb~pksJLal$ZF7Q|luAZ1x4a!U^uW#5J9HhA2Kxd1gUki4YEWUjW7 z;qBnR1MJCNu9E@AEw^(pl)3R9Jb$TQ)y->e=z1+~Ym{Egv~EL;L~Cj4SeKO%K7}7p z!)%0%c4;JbC8$Gy^*k(+;UY*|vdAAlECwT6O?p&MN^F@U6lO5o2dV2`@NZso;B)U( z5An1Fp>18-mZ-rQ_g~Mi5w?6(ZG-4)xoX=5g#v zDaSR_{_BdOONLKfAgUQ-LB3by-AOj@=yka23<%}Plwl6+{t&;IFq-L${5EVv0bG@_ z04z8u@L`z%$zO)vxo(h*lba0duqm#4c(VZJA@{VX`wIua*;jD_y>vTNRjg(}gk=1m zqyg|o*)w9LcoSb#P4lkaq>oS>&{-*Sp!-lF+_}tAw_Y-56<_nZHNNAEBJ7cl&a~D( zn5`8$IJu%tz4BvD%A9hHh4WBJ@sOE+58s_%LMI%Qr}K&<8%{3`S0nguOlEp(9l(x1 z#l&)l9|&Tj>m-CrbSb2r5P>_r&@?B8Zowm6QhcM~%iS5NG{vj12lKrT#0*O{tl-(LY1Vvxw^cYnUw$V^w`0?GxG!uF=x~xovw7Xg0{<=7KxdAyh}wO<_u3@?D(1jUcn8> zKH^uWBFFEPf;tP>qGaB3K_kqrHuDt%=>IV7gcus4yS~j-s4K2bAfL`)E{^TY5#;y0 zN;tWb?kZL%#|&LyNlujGF~CM^#<`Z%uGEJyde|?8r$aLPd)}RTt~bpFDJ}S*Rxa}| zKB5goH4a_#(zR}rcNSLBsT}Gop{#j-Wcvbqjg_`_CZcL;1*({MuEKWDbXOi#d{R77 zV$7H&5n*L#2ezq2((fq2^ADr7wYZ~`rWO#5<@cPc9(0I3wK7H0ikuB$z*J0wEIS=X zVEzK}U~39>^|KGrV!DBX&n~LeA6T!nTU$TV=Al$;ui@|-cld^6vK!L2OGZKat&uRp z{8n5+>9q7Pa=$EFdwF)tK8((6yXtl`a{MGG$0g|TLraJQ#oG51PSfgwDCwM}^Bw)E3g?D;1jLt># z2cTy%oZA%8}{$r;s4R`Yf;nyUBxg=+&2P(_n~ngR~*Nx1z(f z?NrTP1+R3qw4P{5#Dd54Ufd|bc`;D1maZw%{aMxNA%o{ zhDBn_q_RAGWyXZsBzh-;f`^RsrG_-YR9pAJ%Xb?1`10}je*e1?Fd!u29a4TWl1!!@ z(1i{xW6lF7;J)lU0X` z#f_2tNCoeZFQ;10bb9hl!R;D3F!+yY(C@-gCIkHx!f<=}Cu{eXLgCw#ogpAhEDKPY zEWNtVA>32~&=bp!5n5{neAkG(_m{SLAP1dWbc=8*uhWDb5%C*I`dHKai<@=DQA&Hq z4+G&(q0!rqb)<_B#b!-F;qY9X;7+gHo+$puAiUufViyrzG&+q-LV{QBaRzvm9`_sL zaE5>~+Wrk?YA0LgXp7R2(MxYy}X(;xP zz@-Z9#lvSK(*eax9H3%K(zc1Y{@JcR&yiKK)z3U&5Z)~?>zXV_svu*fco>}=FgUJ? zT9i&g-?H!j&^Jha=_-O5hKjex0Coz4k3YhkkpXBT-EVC>=Yfo7mNnA2F5VW}<6~U1 zJG3yiXA{QD8q9=3v^Y2I(X{5nKvmK&&FXLkE%wDjLCOdA}ras=dnz<@wrBRIUd)_KsaDSQlhO6!J!o zFcXc>?(CZe@#1A``_OaL;RKq8<+J(;^RKUjKg=ThRd9FUh-8zK$$l2fB z&`OXqx|lTscA2xLx6_dGh^j233*U?=1_f&ErH!dHKKS0C2}dhwUtWr2wK3N;Os%D9 zhtEpomdyy%CcgrrTbxLRVVdKZYDbDtN6t1$4k^;XVw4!KK4>-j=XR)N(JMjWZ6=(v z&o`n+5r<$yZYMKKX#^TCA!UV)YL`F|`83tPX^KW|1@3V@&ai#xHd^GrQqLY+lmTKR ziwWZ=h^1opxzu+wR?g}01F(XUx;8v5szc;Y-Z(m@cuLX>-g^rQr6i^z8qhcom!3O_ zNeR#KOfMyW>@QB{lhg()_w7+!N-F)-&hw2tRV;=s2@}dG%{K{7Hn8Wti&$Z>LUU_YU`2rvE{f}%4aFkmG zxu$M9CT21v?nGy+64hvDh%WP5$RS2r!T@Be2ugvh*ygTbnEtj220CGkc#fQk2t*$k z#jV@JJL^33nEfEVt;;DqakneTAX%>u+I*dKw2wjxDFJVX$I#mJH{yBRiZzj8NpV(4 zM=Eo?bUXE%#XhZWS5}%>JM-Vt>xah!4L>|MOFn$!pDN`ZNN#!(72(-nF&S+~_!u`G zDt$NR;ZJM|cWkEWzUjdp$d?QVp`d?RGGbpIMi6se z$Up`0=f7g9+nFhw!nJKLII-qSfNYvmhFhGNCVdLZ8yZ4Aw6|D>gsq_Fuol%a2`@CP zzMaQ-N&iNM!EoEtoos;G?Cl12`cU+59duK*7nS=VwMPaYv0g=>fjQfFxyj1ijFd}5 zC5XMRm_^Pz!r|IKDGlzb)dpTv@n2jlW?#|O#(g+FkOU;&9t`S~9#H+L#IT{O)hl_Y z3ysv!o83QvSWvg2(X$u z#%#Mym=^y*;?5@Vy`Zmn@f5ExXy>3#4)om%zsN#Yo6V*3tUPG$yS~g5 zWvyoODm7KQ>#i8FgAbnF6BDVnU+MKdVm0~R*&bp`%n`?3^CI-lX~N(c`E642^D8xw ZaMwnD8<{rj>-VGySO(||l87%HCTDP@jvN30