F-b-|7^X4p`@>(z85@W-SQ-iC7p(9rxyn66<0t+9gM;tEJa(R^!}`(y@?qN7Mut*!-oKtoy%`h(@*%
z!?6)Hql2iWJc^0<1FFM2sD=XWcK>EXq3U_@BxYj>dWzkFgrR1fh?-bBw#EMF)4O{n
zk&d_=)$tn`g}ZSQ9ze}JdAz$qnb?hVE_TN<)WBXqb+i$+lG`u_-^PvjIcg%+6Wq`6
zRTEhMXfn2wp_%W)2>imPzehFvtMvwIX18s5Fn^0xEBGC1>3>r`{hNSNx5H3WM+vB}Tb_7;0csP#w-eHM9_u
zaW!_tt*A5eHL8OjP+RaX)GhOy;eLjDQO{SSPfNa%h?eGk)S>zqb$Sn@W_Aqqz-gR|
zmr;8-d8XUZG~7wL2GwE8EccI37u4;l#Ll=DwWXipGQ2vA^{*##?`-!oyu}>%Gkh6x
zvrI^t`x#z_6G{7*yMG0zqSAlCW9V1me*K=rCrAe}N_~c}!4%TZRl47V@1g#veS@0d
znM$Ag9e$3C?qsCTb7wXMRk0M4a5idYE3hlpVJ_~+JiLKAD;=xc@)@YKgDU?F^YJFC
zA)w_;?Xzq32h8Tsy82Xm>)n*U==a|pLx+{>_Rng05zavs5jdw)POEwI0oG3E?p~YGHRe1n1)?Y
z?Tokir8ZrO0pu^jU|fn3TH8O^f(Fzrc-i_Is^d3N*YP0gdj5c#(RI`SZ=)KHx!?W!
zpMdJ18|t|MsQM)sg0oOdzW|&5{r?ydFBxl46<@Qyg=*+MjKuwzj;Cz;CPtACTj&lT
z6}1BGQ5}uM3@kxCw;UtzRn$uEL7$fH0Fgxe618-fP^bQP)Pt=SxnHY&un*~p*beJ3
z1{+a_>Oama-b_zh=W}n0C)c^)O^9!1Ys|%=
zR0zR0F~>=Y^M*!|>O!hNws#8RGX2YmO?PVIQj!vgZKLcqK|e3)otTVO&epgDPo}d!
zuJg1}r1iT)__Qg;B9gbq#*2twAwGw&*X9jG?M@&2Y%q2ww6N*blGq?JU%nX5UU_B!g(h?tz7`A@dT~v4Y#Vfi}Z5h6{ug3b1Oa}wl^v5L5j`Q
zh9)^Vtus9yXF}`FxzosxBkUr7B96n3wys`*`cwD~;X^_v^4mD?w9YO_C-y!$gHb!G
zUw=Y7(i}%qj2r^v5QY%%K+tbAp)QCp974H^AohLf*(G%ceV{4yX>Xe!6ZwY}pp#
zW0lD-pS-SwCy6%@GKgoXfM1@i-_}i6!RsGM?i?p5v5$Wtv2bU2Vwxw$sYy)n404`M^!6J{ZV|P0N&o*lNaO{=WXkkE
zf!?+SfyDg@euQr14Rp>XW`+(UHJngD$aZ3rGDDja(`)W^LQ7|OQbJHXIeDaRIWs`Oa5rW-A-(B
zTKq_2hbXDCrQ$8ZY-e1uw|f^#TN7dl`c1(Ce3+0*e4R4+wQw=jadmin can publish a dataset that has already been created."
msgstr "You are not authorized to change this field. Only a admin
can publish a dataset that has already been created."
+#: ckanext/schemingdcat/templates/schemingdcat/form_snippets/custom_identifier.html
+msgid "Waiting for Metadata completion..."
+msgstr "Waiting for Metadata completion..."
+
# Themes (NTI-RISP) - Schema field_name: theme_es
msgid "ciencia-tecnologia"
msgstr "Science and technology"
diff --git a/ckanext/schemingdcat/i18n/es/LC_MESSAGES/ckanext-schemingdcat.mo b/ckanext/schemingdcat/i18n/es/LC_MESSAGES/ckanext-schemingdcat.mo
index 67876b23ae5768081756f6884a600fbf21bb51ad..656378979d082656cc2de21d05000c3b343c3dd7 100644
GIT binary patch
delta 7036
zcmXxo33yId9>?+XA|a9>lE|LCL_|cy9vTr_?6Ff~4MHQ4sAa0Qx3#oQTiUv)4z;vQ
zb{p|7?og*9yP-h?2cdKG;D`CUY_9stcLGnRosq#
zxEn)p9|jxaGbc&-Q*aJd@h7Z-SFtXZV;TC@H>Nheg!OPeRyK&04PpW1CsFOSZD33t
z%)$`thfz2V)&3j|q<^!B#9Rtq!pRtz=)N!mtCL@XMYz=F@1Z*OG<2tpY6>4TbVn@7;T8hLZcL15F
zj(TAi9Ehx@Sz*gJAY(UMPy_oCeG2JC5`I{Y)$p$Mp|wgQx4gPF2o>@W{05US2=AjJ
zQ6t$MNIlfV;*sE*cE~m|Be5yYNhbd4_$>;w*6-t^cn~%7$i{>S>ticS#Wpw={c$B$
z$F-<>n^2o{E53n;Q4^ZPPGwt|BGkZ_q9(pEh4^ceY_kQ2Pz`^DdhrL;%+A^8*HNLn
zk9sepiMyr^Fp_+GRKtVt5{}15uzRX81jjs$6qzlk_D}mr=!NgG3jTuK@j3=zOX4&J
zJ76)cLA_Wb&D|5h)<{$&6H(7oPy=j+!I*8$MMZ3+)i=Q=iZGA|GcXu^SQD3`URaGa
za04m=Z=-hc0o2l+LUniqmIjb
z)UjHD3HWzZLnpBzo=0`)m+rn7jjET58h9U6`-4#t9*)~_B(~N0uhrZb(k2IK%gnO*
zO{lf{8kvl#K+QOX354ygCL+UG-2$8D^Az7Q3GGOUfuZ23C$Y2=$p)WsdB
zCHM^0;CWQX<){uHpq3<%QR*BwK=z~QjM_UTs1D{}9ef@`aSaZ^4^Zz{Ypu|;tgVT^
zLNtg1ZLU1jE}npzSs`lV)9^8zW%K7y9bLk0cpKH>WQ~g}F`e8BiMW<*D*EsXBnYNw
zXSe=BoJf8ZF2ws7j|;lE$L)3N2Gj)JwE1nWKC{yne25y^Uexh8Xv_a;J#NcSSx
zM^N>SquTiv)xmk|C0l+4wW)tY)&Bzn719b4dLgizTcH-JLO7~C4%M;O=G$94pazg_
z^F2`=^s)It)}g3lJsdT_>8KCVJoIVp%SfoBb*TJCRKvS%`2p1VKZN=&oU-MYtXHg8
zQ3LqZ=5Jf?SnpXMpf+_yH{!33>*Tm23&qMAqxOOq)nOyl3+=4Eta;WV)O+(#1NEV1
zy3CfZ#=7L!pdz~!6@lG3#9wQ9*j7A^m1}78=TQw_LO(3G<#%oQebip5+1)M~GC31x
z^W!m^{1nsxmtZ}74i(|`J`!5nEvSm0U>cr9z9FV&5BG!A12vEh=y>>);%M?+d%5pz
z#D?VeAwM0=1@xz0aBueqNLU~D*!IWfJTF5{$hVh70EvUBfgD4H>Js|n9juNIQRmmM
zue+H-aV7aQ)Qk>W&!Fnv!dR@pxfq-4cDNe#VOxjnHJ{l>LVw>+VIbbXAiRh3(Z3(z
z!o{ctgZsPnn&8vq+oC402Q|<`sDU3tMeGzRGUrh<_8Z{-Y7Rtyo&Q7k
zFI0#AgWM1|L^a$Ibu4?Jo{zHSb5Ju}gqrbk)C69`K-PXc)*-)tush%rs0dv|FJ42R
zzU6^K+@Hre*n<3IRH$AK!||wr725JrRQ*Mmf~#!)AgY7Yn1KJb
z<+X;h*N9Ag%%D7L1o2NIF>8b|jd2Y|;HRjD&!W!lO&pKqs2L5-V-PqF=ixDIjy*=Y
z@6EQpfLi-?sEBMqE#WrQ1W))#sH2~3g}YYID0jv+QEME6wb6?jKyy^6Gf^FM#bnGy
zMRFEuA}?YHZbEJ1-Ka>OvE{xCBs75Aw!nY18}dk0!%eXXwnarMAHDdj&2O>KkE1%e
zhix%%j9ae@Y7b37ZSr}jQ}ZTP)A|2^ghsRzPvA$W&Gf=pepcZPWcf_5aU4n9hzj)s
z?1gDmj=?8T18^`HH=`oA1GR?^+2`M)BKHec{{8=rghuudHL`{*bY+N94Q65o%*8N#
z9@XG_`+Ntg{t?u1JBgam1^fIL`~1JCCGwl#?u9fAXaAYDB(wy5Q7?`~&1e$F;zH|s
z)PVM(mgYFd;rAGYcd#|qndp9Ka!?T{L$$LSReu+1^PNT?lQvgLsDVBO{3{3xP`mYA
zR7X2e1KV#sf@wIrWoNBjXZF!FJC
z#sg68EW`HrBlg6I>F)3UaXu1VDcFF1co`M?YnXtyQK1f<;f5{+wMmEJejJZ)VEviy
z{~LUYTFSs##xQA5
zMeU6?7>wE25OY!WrrYQ9u?hKSF$lM#CUgM3I{)90(2Vb(j#mY0Glk7|^NH4Ws1fI)
zmTCmn!+ea!*{Js%)WF_A)q4*$pqY+y30M$@C
zYbR6(Juwmop=Mr$n!wAb32nCdt*G`7V-y}kMdGsZI{&vwsKLNF?t^etLk+MlW?%&N
zMm0Ph74jm~jGsge_&IEWD^Tx!hSB%~Y5@PiM7)oBKVdHMZ%QJKL@MTC9ef7Wz^kZ%
zy@N@(6SWu4U_73;go?->)S7>8pMQk|$e%$)Ds_Q-d@~mGaCher
z3e=$w72;JGgX^syVFvl{Fc)jHZbdj66{(|6KtzTo-{~BY-Xf9@jjniX!@bhYR{np1
zmSK>6uHtXF|L1IsNcJ>zjz%PSS~*uEVxp?q8dI=4HEQ8jOn2fUBZ9-orIYJ~O`Oh=
z$$pQK8s|)nOo)pk^)}B8>Wac`=*6kd+mSJz6z51}`sn`lxd%V3tjWLPN$C}%&^>wzTnW%k{YV$2nJ6)SmmrhYT`)o7m{wm~ZPg!g3=SjcH-I#P^J>Y6<
z>(_U)W-xiSp-IB&Smfb9Y(6CM1outu``o2Gyn+w7b%i@OVj}!PDV^j5#Af<+BNgKG
zicR#?bEd{7cygSju@PClD9xd^&eWgR7bO15oyW6j+&k@qnxw09`*F9Vtc!CYHaWOA
zxjx)oxzn8Rxa45{D^#D!x446xUU4x2`UT#KT!k|=E;D5yxyjtMx$jZ>ICn8=eaS9x
z*CG8Tc63g~#iWcNr<49DE<#;N0e>Uj<
DJ1kq@
delta 6942
zcmXxp2Xs|M9>?){kU&Bb5>iMnl;ja0CLxf}OCU%H5NaSsibCjJS(Xx>6bm9!F3k;1
zIpRU-AP9?qAcC&4N?8yrh>J&Y5kXywg8TjD&fc@v&&=GpQ~on^AFyBi>G|WK$M;pZ
z-%7(Z$zx0ct_U*bN7BL3DmCVvBx8#407hb9vN3v;gz1=z=WrOd#1<*WJc1L^57%LJ
zd=33^Ge+Q63^B%Mj*_TB!D&>*^B91iV>n*Lh4>qWqOYDYb#Vz+HHcLWVmakUQSGJG
zHzo`-unxAzSnQ2ze=G*kzj=zpEDC%$3V%nvFuZ{=HE|k_!wQ@K0oCy>>m6h!<^jfI
zP($~*7o*5$pxWt-8h8<^{U{@QG_pDN!5XYh{!P?K4`3}khML(Y*dEWJmLek6
z9Y7OQN3Ad)J0YuS7TEHo$QaEU)W8m*Pa!=+!XK}qAAWDWVZCj=WBmgaa!(q|g;5xc
zKcXUW7d4O?UUy=(k)WDJ$Tl)X*c8WliN8OI)f8y0*Wp0iiJEypBSM6AFbCtY1r}os
zd=_isOQ?D)FcR0}Yq$qBp%Qi~+rpHg23~=h__B23uTAo*E!d4}_!#QN4^cBaZJ%F8
zh3-exd!EMbnucQx`E*pn`S>mN#nzbHgbj&PkRr1N)&6lG3B7O%tKmiLikC4E>l3FT
z*c8X(BGij_u@*kC24uKP7=d~oiyB}OhG3?(9V%i)R$o7xD8nEg495_hgzBII^}-7n
zfJ;#kcm=hKcc7N;JyeJ1P)l$PHIVzL&HN{7qP6)rD4&GneI|p1Dz-zNf_zj%kE4#u
z1k|xwfP7=jMpQ#bu_1nn>hL$zd$ls%dhw`%w?Vbv85QAfxEqTwSLgpB3I1nVWVv6i
za#Vf=YOUT!25)YnW}KMq?ukrmTMVJR5Vdy(qLyw5Y5=1#7$>9lNChggtEA5Vt0d~+
zR@5#(Z1cBKBlm0OHW+A)z}l22qB?AfdOycLFF+l)V*7k5s+}1aigRrF%jna{SCRNGB-09HGl)C$or0vh$L|mBk&9R;AdP*{x14(Wk-HkFshSV
zuNX&=ACB|z6eeL&zI)8ZS|^|eKH28ySm(O>%mNY`(etQdvDj8vW39C1>#bX?d#y*T
zA6d_1IPF})6#NDC;fn0+4lEIM98)k*Q_CQs3VEmjbVP-w2&1qc#^4y#rkjVVzZ_Ms
z5_L*8VKv-l-D%7Bpf>ZnsQT}tPRqwwO`*MLD_ph}uId5h*HIncu=yHY-10!w0D^5k
z3e`c3%_m#yqmFYbYJeS3GcQCfc?tT|(KuT$5!LW~RQXG&bH4=jJy>tccUt#Y_n`)G
zz~+xyk6S;meu&!4AKUzwU5LL%cEuK4N9~0hs1AQYz2INqTG#5ewnPoI5H-*qsP@Wi
zc{yq?j7Eii7AgYsQA@eBfcUF{jTBU^Au7KO)!;h4$mgNj>x*@904l=cQA<0`M?w`}#thtod^^kq9E6dNx&xVjj)!xOgDDRybl;nZ
z4aqM={>WgqV-4zELcUGro9^zhO(-%Zi+lc#F1o<$9G32NZ0QA@WT6`5_Q0i3~TyojpzGuGAlze7S1sNLJ$
zWL``p-x3>O38vsoRD>!~1KEyR%Y!%&kE3QD-^bltEm0xvhMZv27d63EsP{Hun$G`L
z66)|QD#W*Jg`mD}s3K9%(@`%zike|>)J%q=CNL6%SbHCakzdr$9q?LIgmz#G?niyf
zFQWc7Wy1TrKPJslp&E&5a6FF3XHmQTHfrV%um}SO81n=UKy|PmHNYdN`lnEP!{6M$xo=3W?u>MSB1+IgyU_DLVw~Jf$^vrW}*g`Ys=fB>i5QU9B%WAQ5|f=Wc<4=
zzl`xj<|oXfyjCe+Ma(ZH{*6eCrXU)Zp&H(TI=6@LNj!p@QT@mG9l#8njjJ&WBg@?P
z3ao=rYd;PZk!h$UoP(O+T2w!~eYV00>uJ=CFQC@=D-6XOr~y1cg*xDIw}UYBl8-}0
zG9NXOQmlhhP@8x@s@*qi`F7L*d`E4;Syafcp+8Pqmdn%~W(-j_6H%f5
z5DW1>#$y*=(g22G2u?*s?itjCme}W;P?38ZbqwFdG@bvCNN8lYuqwo;1_N05wit(z
zI1ts~c+~S}Q1zFij@vp^h_~D4Z`0NwHNNA_CWRFECKzSSQ2`%DQZSJn1IFB
z@u&_LqL!u-6Y+J7#pBoyIn~ZV)_W-!4gzRr=yl+1-8e{n1|O;Gftf1
zwo`_!$iIo*@ZZ=0Gp4#rHNi*1pMqVe(C^1&Jc<%?Vt9In&&|^bTr?&!O7+3G3i}Oh;evU)&JpqQ3ddQ3HAlwYzuO@XVhhhp&LrruQM(X@;AfZjP*DWx|trt-vzJXezdsr7e
ze|5j{F{t-)P#x!^>h(Yks6S@mP*gihQT5hh4cv?YI{({jg?*@z9z!*B(Ru~-;`bPX
zw^1_>n&EyIvQRTBu=!r74u)eajzvXcfz2;NwfC0F>EFCff>4-a7>*Y(8n2@ot~S#R
zc@S#Gv8Wj}!fb4gdT%7g;SAIOR$v3HL@n77Y>J;?6TFYUFcN9A+y?ScBkPW-*dMhQ
zW?~XPW6L*SI{6dW32&kovuC>>no)4=&WI@#0AxfdNDTiwF5;+4Q5NZtW
zt$jBbbGZX-ej(4|xu0;Jh)s@q(w0(Cbv;hLm9sL|8|Wc5lCnC^iP#3IHOUR&ZbQjk
zx2h>3uY)ieM{w72LgM11+K|$AO+j7yX-jh2#d$r|obtHL>>-rLa_^>mAZo!g@D|SH
zPUODCeV99w@_NpRxF(%aNgbr500*M3cHC*?`KFrwsB1ZQSJF*vUb~<@>BpSx_$I!N
zwzM^BUo^IP{n%)yYcuN7Day0YUMF3wLavsS<#4}9dNsF~bR#|BYGLcwbF-!|dA6Nt
zh*NQ_hkxDMPhvXvP43^hC-U$+{DWIpr1Ky?+CQAqawj&SjeloSVNPj615cDwk&x`^
z;;c)EZr6>{0&43p{{K2f;&0rgJk$RvwAVh+KWpi4mVVsLDC^|hO7MmhlIzZ$&)w8X
zP4tHNlhSAM4elVPG%-Ffj*=X551op{HtD^|jpVM)eUH*9+~Y{=OZE+SDCteu&iN)W
zK79~5o%9l%kGkIBezhva*PilX?tV^cQUgyPFt}^V1d`?Y$(koC`
bnv0o0+2B3fk_Obadmin can publish a dataset that has already been created."
msgstr "No está autorizado a modificar este campo. Solo un administrador
puede publicar un conjunto de datos ya creado."
+#: ckanext/schemingdcat/templates/schemingdcat/form_snippets/custom_identifier.html
+msgid "Waiting for Metadata completion..."
+msgstr "Esperando a la finalización del Metadato..."
+
# Themes (NTI-RISP) - Schema field_name: theme_es
msgid "ciencia-tecnologia"
msgstr "Ciencia y tecnología"
diff --git a/ckanext/schemingdcat/templates/schemingdcat/form_snippets/custom_identifier.html b/ckanext/schemingdcat/templates/schemingdcat/form_snippets/custom_identifier.html
new file mode 100644
index 0000000..9470e41
--- /dev/null
+++ b/ckanext/schemingdcat/templates/schemingdcat/form_snippets/custom_identifier.html
@@ -0,0 +1,14 @@
+
\ No newline at end of file
From e606499c8f3300d15fda9b20446dfff371478ac1 Mon Sep 17 00:00:00 2001
From: mjanez <96422458+mjanez@users.noreply.github.com>
Date: Wed, 13 Nov 2024 00:51:30 +0100
Subject: [PATCH 3/5] Add validators and presets for tag_string normalization
(normalize_tag_string_autocomplete)
---
ckanext/schemingdcat/config/__init__.py | 1 +
ckanext/schemingdcat/config/tools.py | 1 +
.../schemingdcat/schemas/default_presets.json | 31 ++++++++
ckanext/schemingdcat/validators.py | 74 ++++++++++++++++++-
4 files changed, 105 insertions(+), 2 deletions(-)
diff --git a/ckanext/schemingdcat/config/__init__.py b/ckanext/schemingdcat/config/__init__.py
index 86382e9..a231dac 100644
--- a/ckanext/schemingdcat/config/__init__.py
+++ b/ckanext/schemingdcat/config/__init__.py
@@ -54,6 +54,7 @@
'slugify_pat',
'URL_REGEX',
'INVALID_CHARS',
+ 'TAGS_NORMALIZE_PATTERN',
'ACCENT_MAP',
'COMMON_DATE_FORMATS'
]
\ No newline at end of file
diff --git a/ckanext/schemingdcat/config/tools.py b/ckanext/schemingdcat/config/tools.py
index 8828b78..1e344fb 100644
--- a/ckanext/schemingdcat/config/tools.py
+++ b/ckanext/schemingdcat/config/tools.py
@@ -26,6 +26,7 @@
# Compile the regular expression
INVALID_CHARS = re.compile(r"[^a-zñ0-9_.-]")
+TAGS_NORMALIZE_PATTERN = re.compile(r'[^a-záéíóúüñ0-9\-_\.]')
# Define a dictionary to map accented characters to their unaccented equivalents except ñ
ACCENT_MAP = str.maketrans({
diff --git a/ckanext/schemingdcat/schemas/default_presets.json b/ckanext/schemingdcat/schemas/default_presets.json
index 962fb06..96e1220 100644
--- a/ckanext/schemingdcat/schemas/default_presets.json
+++ b/ckanext/schemingdcat/schemas/default_presets.json
@@ -39,6 +39,19 @@
}
}
},
+ {
+ "preset_name": "normalize_tag_string_autocomplete",
+ "values": {
+ "validators": "ignore_missing normalize_tag_strings tag_string_convert",
+ "classes": ["control-full"],
+ "form_attrs": {
+ "data-module": "autocomplete",
+ "data-module-tags": "",
+ "data-module-source": "/api/2/util/tag/autocomplete?incomplete=?",
+ "class": ""
+ }
+ }
+ },
{
"preset_name": "tag_string_uris",
"values": {
@@ -307,6 +320,24 @@
"output_validators": "scheming_load_json"
}
},
+ {
+ "preset_name": "required_multiple_text_raws_ordered",
+ "values": {
+ "form_snippet": "schemingdcat/form_snippets/multiple_text.html",
+ "display_snippet": "schemingdcat/display_snippets/list_raws_ordered.html",
+ "validators": "not_empty scheming_required schemingdcat_multiple_text",
+ "output_validators": "scheming_load_json"
+ }
+ },
+ {
+ "preset_name": "required_multiple_text_links",
+ "values": {
+ "form_snippet": "schemingdcat/form_snippets/multiple_text.html",
+ "display_snippet": "schemingdcat/display_snippets/list_links.html",
+ "validators": "not_empty scheming_required schemingdcat_multiple_text",
+ "output_validators": "scheming_load_json"
+ }
+ },
{
"preset_name": "markdown",
"values": {
diff --git a/ckanext/schemingdcat/validators.py b/ckanext/schemingdcat/validators.py
index a953d6c..35fb335 100644
--- a/ckanext/schemingdcat/validators.py
+++ b/ckanext/schemingdcat/validators.py
@@ -3,6 +3,7 @@
import six
import mimetypes
from shapely.geometry import shape, Polygon
+from functools import lru_cache
import ckanext.scheming.helpers as sh
import ckanext.schemingdcat.helpers as helpers
@@ -35,7 +36,8 @@
from ckanext.schemingdcat.config import (
OGC2CKAN_HARVESTER_MD_CONFIG,
mimetype_base_uri,
- DCAT_AP_HVD_CATEGORY_LEGISLATION
+ DCAT_AP_HVD_CATEGORY_LEGISLATION,
+ TAGS_NORMALIZE_PATTERN
)
log = logging.getLogger(__name__)
@@ -1158,4 +1160,72 @@ def validator(key, data, errors, context):
if data.get(key) != DCAT_AP_HVD_CATEGORY_LEGISLATION:
data[key] = [DCAT_AP_HVD_CATEGORY_LEGISLATION]
- return validator
\ No newline at end of file
+ return validator
+
+@scheming_validator
+@validator
+def normalize_tag_strings(field, schema):
+ """
+ Normalizes the value of a specified tag_string and tags before tag_string_convert validator using the rules determined by normalize_string
+
+ Args:
+ field (dict): Information about the field to update.
+ schema (dict): The schema for the field to update.
+
+ Returns:
+ function: A validation function to normalize the value of the key.
+ """
+ log.debug('miteco_normalize_tag_string: %s', field)
+
+ def validator(key, data, errors, context):
+ value = data.get(key)
+
+ try:
+ if value:
+ normalized_values = []
+ if isinstance(value, str):
+ tags = value.split(',')
+ normalized_values = [normalize_string(tag.strip()) for tag in tags]
+ data[key] = ','.join(normalized_values)
+ elif isinstance(value, list):
+ for tag in value:
+ if 'name' in tag:
+ tag['name'] = normalize_string(tag['name'].strip())
+ if 'display_name' in tag:
+ tag['display_name'] = normalize_string(tag['display_name'].strip())
+
+ # Normalize the tags in data
+ for data_key in data.keys():
+ if isinstance(data_key, tuple) and data_key[0] == 'tags' and data_key[2] == 'name':
+ data[data_key] = normalize_string(data[data_key].strip())
+
+ except Exception as e:
+ log.error(f"Error normalizing tags: {e}")
+
+ return validator
+
+@staticmethod
+@lru_cache(maxsize=44)
+def normalize_string(s):
+ """Normalizes a string according to the rules:
+ - Replaces spaces with hyphens.
+ - Converts to lowercase.
+ - Removes disallowed characters.
+ - Normalize to using only alphanumeric and spanish accents (áéíóúüñ) or hyphens "-", underscores "_" and dots "."
+ - Limits the length to 30 characters.
+
+ Args:
+ s (str): String to normalize.
+
+ Returns:
+ str: Normalized string.
+
+ Raises:
+ Invalid: If the string contains disallowed characters.
+ """
+ s = s.strip()
+ s = s.lower()
+ s = s.replace(' ', '-')
+ s = TAGS_NORMALIZE_PATTERN.sub('', s)
+ s = s[:30]
+ return s
\ No newline at end of file
From ad8172dae84714f970057fe516a8d75f40ed9865 Mon Sep 17 00:00:00 2001
From: mjanez <96422458+mjanez@users.noreply.github.com>
Date: Wed, 13 Nov 2024 00:57:08 +0100
Subject: [PATCH 4/5] Add private_fields logic to package_show, package_search,
etc. and config options
- Improve before_dataset_search and after_dataset_search to avoid exposing private_fields
- Improve dataset_show to clean up private_fields
- Add options to config_declaration to configure private_fields and roles that have access to private_fields.
- Improve docs.
---
ckanext/schemingdcat/config_declaration.yml | 19 ++
ckanext/schemingdcat/helpers.py | 1 -
ckanext/schemingdcat/package_controller.py | 188 +++++++++++++++-----
ckanext/schemingdcat/plugin.py | 1 +
ckanext/schemingdcat/utils.py | 23 ++-
docs/v1/configuration.md | 26 +++
6 files changed, 211 insertions(+), 47 deletions(-)
diff --git a/ckanext/schemingdcat/config_declaration.yml b/ckanext/schemingdcat/config_declaration.yml
index a7a94bb..d0db8a2 100644
--- a/ckanext/schemingdcat/config_declaration.yml
+++ b/ckanext/schemingdcat/config_declaration.yml
@@ -58,6 +58,25 @@ groups:
required: false
example: 'https://demo.pycsw.org/cite/csw'
+ - annotation: API settings
+ options:
+ - key: ckanext.schemingdcat.api.private_fields
+ description: |
+ List of fields that should not be exposed in the API actions like `package_show`, `package_search` `resource_show`, etc.
+ type: list
+ default: []
+ required: false
+
+ - key: ckanext.schemingdcat.api.private_fields_roles
+ description: |
+ List of members that has access to private_fields. By default members of the organization with the role `admin`, `editor` and `member` have access to private fields.
+ type: list
+ default:
+ - admin
+ - editor
+ - member
+ required: false
+
- annotation: Facet settings
options:
- key: ckanext.schemingdcat.default_facet_operator
diff --git a/ckanext/schemingdcat/helpers.py b/ckanext/schemingdcat/helpers.py
index e4d55bf..3c54d39 100644
--- a/ckanext/schemingdcat/helpers.py
+++ b/ckanext/schemingdcat/helpers.py
@@ -1990,7 +1990,6 @@ def schemingdcat_user_is_org_member(
>>> schemingdcat_user_is_org_member("org_id", user, "editor")
True
"""
- log.debug(f"{locals()=}")
result = False
if org_id is not None and user is not None:
member_list_action = p.toolkit.get_action("schemingdcat_member_list")
diff --git a/ckanext/schemingdcat/package_controller.py b/ckanext/schemingdcat/package_controller.py
index 2ca474d..6d2061f 100644
--- a/ckanext/schemingdcat/package_controller.py
+++ b/ckanext/schemingdcat/package_controller.py
@@ -7,6 +7,7 @@
)
import ckanext.schemingdcat.helpers as sdct_helpers
+from ckanext.schemingdcat.utils import remove_private_keys
import logging
import sys
@@ -47,11 +48,12 @@ def delete(self, entity):
def before_search(self, search_params):
return self.before_dataset_search(search_params)
- # CKAN >= 2.10
def before_dataset_search(self, search_params):
- """Modifies search parameters before executing a search.
+ """
+ Modifies search parameters before executing a search.
- This method adjusts the 'fq' (filter query) parameter based on the 'facet.field' value in the search parameters. If 'facet.field' is a list, it iterates through each field, applying the '_facet_search_operator' to modify 'fq'. If 'facet.field' is a string, it directly applies the '_facet_search_operator'. If 'facet.field' is not present or is invalid, no modification is made.
+ This method adjusts the 'fq' (filter query) parameter based on the 'facet.field' value in the search parameters.
+ It also removes private fields from 'fl' parameters.
Args:
search_params (dict): The search parameters to be modified. Expected to contain 'facet.field' and 'fq'.
@@ -62,8 +64,19 @@ def before_dataset_search(self, search_params):
Raises:
Exception: Captures and logs any exception that occurs during the modification of search parameters.
"""
- try:
- #log.debug("Initial search_params: %s", search_params)
+ try:
+ private_fields = p.toolkit.config.get('ckanext.schemingdcat.api.private_fields', [])
+
+ # Ensure private_fields is a list of strings
+ if not isinstance(private_fields, list) or not all(isinstance(field, str) for field in private_fields):
+ private_fields = []
+
+ # Clean 'fl' parameter
+ if 'fl' in search_params and search_params['fl'] is not None:
+ fl_fields = search_params['fl']
+ fl_fields = [field for field in fl_fields if field not in private_fields and not any(field.startswith(f'extras_{pf}') for pf in private_fields)]
+ search_params.update({'fl': fl_fields})
+
facet_field = search_params.get('facet.field', '')
#log.debug("facet.field: %s", facet_field)
@@ -79,7 +92,7 @@ def before_dataset_search(self, search_params):
if new_fq and isinstance(new_fq, str):
search_params.update({'fq': new_fq})
except Exception as e:
- log.error("[before_search] Error: %s", e)
+ log.error("[before_dataset_search] Error: %s", e)
return search_params
# CKAN < 2.10
@@ -87,6 +100,34 @@ def after_search(self, search_results, search_params):
return self.after_dataset_search(search_results, search_params)
def after_dataset_search(self, search_results, search_params):
+ """
+ Process the search results after a search, efficiently removing private keys.
+
+ Args:
+ search_results (dict): The search results dictionary to be processed.
+ search_params (dict): The search parameters used for the search.
+
+ Returns:
+ dict: The processed search results dictionary with private keys removed from each result.
+ """
+ try:
+ private_fields = p.toolkit.config.get('ckanext.schemingdcat.api.private_fields', [])
+
+ # Ensure private_fields is a list of strings
+ if not isinstance(private_fields, list) or not all(isinstance(field, str) for field in private_fields):
+ private_fields = []
+
+ # Precompute the set of fields to remove, including 'extras_' prefixed fields
+ fields_to_remove = set(private_fields + [f"extras_{field}" for field in private_fields])
+
+ # Process each result in the search results
+ for result in search_results.get('results', []):
+ for field in fields_to_remove:
+ result.pop(field, None) # Removes the field if it exists
+
+ except Exception as e:
+ log.error("[after_dataset_search] Error: %s", e)
+
return search_results
# CKAN < 2.10
@@ -116,7 +157,58 @@ def before_dataset_index(self, data_dict):
data_dict = self._before_index_dump_dicts(data_dict)
return data_dict
+
+ # CKAN < 2.10
+ def before_view(self, pkg_dict):
+ return self.before_dataset_view(pkg_dict)
+
+ def before_dataset_view(self, pkg_dict):
+ return pkg_dict
+
+ # CKAN < 2.10
+ def after_create(self, context, data_dict):
+ return self.after_dataset_create(context, data_dict)
+
+ def after_dataset_create(self, context, data_dict):
+ return data_dict
+
+ # CKAN < 2.10
+ def after_update(self, context, data_dict):
+ return self.after_dataset_update(context, data_dict)
+
+ def after_dataset_update(self, context, data_dict):
+ return data_dict
+
+ # CKAN < 2.10
+ def after_delete(self, context, data_dict):
+ return self.after_dataset_delete(context, data_dict)
+
+ def after_dataset_delete(self, context, data_dict):
+ return data_dict
+
+ # CKAN < 2.10 hooks
+ def after_show(self, context, data_dict):
+ return self.after_dataset_show(context, data_dict)
+
+ def after_dataset_show(self, context, data_dict):
+ """
+ Process the dataset after it is shown, removing private keys if necessary.
+
+ Args:
+ context (dict): The context dictionary containing user and other information.
+ data_dict (dict): The dataset dictionary to be processed.
+ Returns:
+ dict: The processed dataset dictionary with private keys removed if necessary.
+ """
+ data_dict = self._clean_private_fields(context, data_dict)
+
+ return data_dict
+
+ def update_facet_titles(self, facet_titles):
+ return facet_titles
+
+ # Additional methods
def convert_stringified_lists(self, data_dict):
"""
Converts stringified lists in the data dictionary to actual lists.
@@ -251,44 +343,6 @@ def _before_index_dump_dicts(self, data_dict):
data_dict[key] = json.dumps(value)
return data_dict
- # CKAN < 2.10
- def before_view(self, pkg_dict):
- return self.before_dataset_view(pkg_dict)
-
- def before_dataset_view(self, pkg_dict):
- return pkg_dict
-
- # CKAN < 2.10
- def after_create(self, context, data_dict):
- return self.after_dataset_create(context, data_dict)
-
- def after_dataset_create(self, context, data_dict):
- return data_dict
-
- # CKAN < 2.10
- def after_update(self, context, data_dict):
- return self.after_dataset_update(context, data_dict)
-
- def after_dataset_update(self, context, data_dict):
- return data_dict
-
- # CKAN < 2.10
- def after_delete(self, context, data_dict):
- return self.after_dataset_delete(context, data_dict)
-
- def after_dataset_delete(self, context, data_dict):
- return data_dict
-
- # CKAN < 2.10
- def after_show(self, context, data_dict):
- return self.after_dataset_show(context, data_dict)
-
- def after_dataset_show(self, context, data_dict):
- return data_dict
-
- def update_facet_titles(self, facet_titles):
- return facet_titles
-
def package_controller_config(self, default_facet_operator):
self.default_facet_operator = default_facet_operator
@@ -330,4 +384,48 @@ def _facet_search_operator(self, fq, facet_field):
# In case of error, return the original fq
new_fq = fq
- return new_fq
\ No newline at end of file
+ return new_fq
+
+ def _clean_private_fields(self, context, data_dict):
+ """
+ Process the dataset after it is shown, removing private keys if necessary.
+
+ Args:
+ context (dict): The context dictionary containing user and other information.
+ data_dict (dict): The dataset dictionary to be processed.
+
+ Returns:
+ dict: The processed dataset dictionary with private keys removed if necessary.
+ """
+ private_fields_roles = p.toolkit.config.get('ckanext.schemingdcat.api.private_fields_roles')
+
+ # Ensure private_fields_roles is a list of strings
+ if not isinstance(private_fields_roles, list) or not all(isinstance(role, str) for role in private_fields_roles):
+ private_fields_roles = ['admin']
+
+ try:
+ user = context.get("auth_user_obj")
+ if user is None or user.is_anonymous:
+ data_dict = remove_private_keys(data_dict)
+ return data_dict
+
+ if hasattr(user, 'sysadmin') and user.sysadmin:
+ return data_dict
+
+ if data_dict is not None:
+ org_id = data_dict.get("owner_org")
+ if org_id is not None:
+ members = p.toolkit.get_action("schemingdcat_member_list")(
+ data_dict={"id": org_id, "object_type": "user"}
+ )
+ for member_id, _, role in members:
+ if member_id == user.id and role.lower() in private_fields_roles:
+ return data_dict
+ data_dict = remove_private_keys(data_dict)
+ else:
+ data_dict = remove_private_keys(data_dict)
+ except Exception as e:
+ log.error('Error in after_dataset_show: %s', e)
+ data_dict = remove_private_keys(data_dict)
+
+ return data_dict
\ No newline at end of file
diff --git a/ckanext/schemingdcat/plugin.py b/ckanext/schemingdcat/plugin.py
index dd3526f..18c3e95 100644
--- a/ckanext/schemingdcat/plugin.py
+++ b/ckanext/schemingdcat/plugin.py
@@ -50,6 +50,7 @@ class SchemingDCATPlugin(
p.implements(p.IConfigurer)
p.implements(p.ITemplateHelpers)
p.implements(p.IFacets)
+ # Custom PackageController, also remove private keys from the package dict
p.implements(p.IPackageController)
p.implements(p.ITranslation)
p.implements(p.IValidators)
diff --git a/ckanext/schemingdcat/utils.py b/ckanext/schemingdcat/utils.py
index aa48d79..05377af 100644
--- a/ckanext/schemingdcat/utils.py
+++ b/ckanext/schemingdcat/utils.py
@@ -621,4 +621,25 @@ def sql_clauses(schema, table, column, alias):
return f"ST_SRID({schema}.{table}.{column}) AS {alias}"
else:
- return f"{schema}.{table}.{column} AS {alias}"
\ No newline at end of file
+ return f"{schema}.{table}.{column} AS {alias}"
+
+def remove_private_keys(data, private_keys=None):
+ """
+ Removes private keys from a dictionary.
+
+ Args:
+ data (dict): The dictionary from which private keys will be removed.
+ private_keys (list, optional): A list of private keys to remove. If not provided, uses DEFAULT_PRIVATE_KEYS.
+
+ Returns:
+ dict: The dictionary without the private keys.
+ """
+ if private_keys is None:
+ private_keys = p.toolkit.config.get('ckanext.schemingdcat.api.private_fields')
+
+ #log.debug('private_keys: %s', private_keys)
+ for key in private_keys:
+ if key in data:
+ del data[key]
+ #log.debug('Processed data: %s', data)
+ return data
\ No newline at end of file
diff --git a/docs/v1/configuration.md b/docs/v1/configuration.md
index 4434d6b..3e7f7b7 100644
--- a/docs/v1/configuration.md
+++ b/docs/v1/configuration.md
@@ -125,6 +125,32 @@ Base URI for spatial CSW Endpoint. By default `/csw` is used, provided it is use
+### API settings
+
+
+#### ckanext.schemingdcat.api.private_fields
+
+
+
+
+
+List of fields that should not be exposed in the API actions like `package_show`, `package_search` `resource_show`, etc.
+
+
+
+#### ckanext.schemingdcat.api.private_fields_roles
+
+
+
+
+Default value: `['admin', 'editor', 'member']`
+
+
+List of members that has access to private_fields. By default members of the organization with the role `admin`, `editor` and `member` have access to private fields.
+
+
+
+
### Facet settings
From 404b83887e35d817a9e4c806865fe0fb2032fae0 Mon Sep 17 00:00:00 2001
From: mjanez <96422458+mjanez@users.noreply.github.com>
Date: Wed, 13 Nov 2024 01:54:41 +0100
Subject: [PATCH 5/5] Fix user organization membership check and clean up tag
normalization logging
---
ckanext/schemingdcat/helpers.py | 7 ++++---
ckanext/schemingdcat/validators.py | 2 --
2 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/ckanext/schemingdcat/helpers.py b/ckanext/schemingdcat/helpers.py
index 3c54d39..622d37d 100644
--- a/ckanext/schemingdcat/helpers.py
+++ b/ckanext/schemingdcat/helpers.py
@@ -1990,14 +1990,15 @@ def schemingdcat_user_is_org_member(
>>> schemingdcat_user_is_org_member("org_id", user, "editor")
True
"""
+ if not user or not hasattr(user, 'id'):
+ return False
+
result = False
- if org_id is not None and user is not None:
+ if org_id is not None:
member_list_action = p.toolkit.get_action("schemingdcat_member_list")
org_members = member_list_action(
data_dict={"id": org_id, "object_type": "user"}
)
- #log.debug(f"{user.id=}")
- #log.debug(f"{org_members=}")
for member_id, _, member_role in org_members:
if user.id == member_id:
#log.debug('member_role: %s and role: %s', member_role, role)
diff --git a/ckanext/schemingdcat/validators.py b/ckanext/schemingdcat/validators.py
index 35fb335..185d2c6 100644
--- a/ckanext/schemingdcat/validators.py
+++ b/ckanext/schemingdcat/validators.py
@@ -1175,8 +1175,6 @@ def normalize_tag_strings(field, schema):
Returns:
function: A validation function to normalize the value of the key.
"""
- log.debug('miteco_normalize_tag_string: %s', field)
-
def validator(key, data, errors, context):
value = data.get(key)