From 404137770f362dbc4aa012fa66a2dba954ed2774 Mon Sep 17 00:00:00 2001 From: Yelizar Alturmessov Date: Thu, 18 Mar 2021 10:47:19 -0400 Subject: [PATCH 1/8] Prepare mb/5.4.0-SNAPSHOT --- pom.xml | 2 +- score-client/pom.xml | 2 +- score-core/pom.xml | 2 +- score-fs/pom.xml | 2 +- score-server/pom.xml | 2 +- score-test/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index b07018ae..882beeb5 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0-SNAPSHOT pom ${project.artifactId} diff --git a/score-client/pom.xml b/score-client/pom.xml index dae999b6..4de048ab 100644 --- a/score-client/pom.xml +++ b/score-client/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0-SNAPSHOT ../pom.xml diff --git a/score-core/pom.xml b/score-core/pom.xml index 6f8c6910..4a188eff 100644 --- a/score-core/pom.xml +++ b/score-core/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0-SNAPSHOT ../pom.xml diff --git a/score-fs/pom.xml b/score-fs/pom.xml index e8c4d1f0..66c4b9b3 100644 --- a/score-fs/pom.xml +++ b/score-fs/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0-SNAPSHOT ../pom.xml diff --git a/score-server/pom.xml b/score-server/pom.xml index d744470d..52763f2c 100644 --- a/score-server/pom.xml +++ b/score-server/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0-SNAPSHOT ../pom.xml diff --git a/score-test/pom.xml b/score-test/pom.xml index 5a972173..66b333fe 100644 --- a/score-test/pom.xml +++ b/score-test/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.3.0 + 5.4.0-SNAPSHOT ../pom.xml From 54a4b68af988f2a76bc26b13d6aa7eea33f8e826 Mon Sep 17 00:00:00 2001 From: Dusan Andric Date: Tue, 11 May 2021 11:35:55 -0400 Subject: [PATCH 2/8] azure setup documentation --- AZURE.md | 44 ++++++++++++++++++++++++++++++++++++ docs/img/azure_policies.png | Bin 0 -> 9907 bytes 2 files changed, 44 insertions(+) create mode 100644 AZURE.md create mode 100644 docs/img/azure_policies.png diff --git a/AZURE.md b/AZURE.md new file mode 100644 index 00000000..de5b07ef --- /dev/null +++ b/AZURE.md @@ -0,0 +1,44 @@ +# Azure Documentation + +## Server Setup + +### Storage Container Setup +Assuming you have a storage account already created inside a resource group +in Azure, create a container called `data`. + +Upload a sentinel object into the container. This can be a small text file that +score will use to validate connectivity to the object storage at request time. + +Create access policies on the storage container. One for read operations and one +for write operations. +Example: +![Access Policies](./docs/img/azure_policies.png) + +Configure the score server either in the config file or env vars: + +```yml +object: + sentinel: # name of the sentinel object you upoaded + +azure: + endpointProtocol: https + accountName: + accountKey: + +bucket: + name.object: data # container name + policy.upload: # name of the policy for the write/add/modify operations + policy.download: # name of policy for the read/list operations + +download: + partsize: 250000000 # safe default +``` + +Start the server with the following run profiles enabled: `prod,secure,azure` + +## Client Setup +Follow the regular client setup of access token and storage and metadata urls. +However when running the client use azure pofile option: +```bash +$ bin/score-client --profile azure ... +``` \ No newline at end of file diff --git a/docs/img/azure_policies.png b/docs/img/azure_policies.png new file mode 100644 index 0000000000000000000000000000000000000000..a06ff45e787bdec4c2353985f50e2d6b036434ab GIT binary patch literal 9907 zcmdUVbyQT}*Ec8<(jYB8NJ$Re3eq8h$cTXC2uODfAYD3i4h$vTN)06?T|>8klyo=q zjz9hW-sj)nde-|s>zQ@d+q=o~Xw$cIJ|req$2V z{uJK*@fss0Vpa40%Uy-%_1bu^?Y?Hd{6?3Fdm1n-F2WJ&5V1tSwm=~6+}M_qs9!H4 zy&)m^J}qM)Xz7v_lT7`=hq{QNt|;L2(AnwujLYFBOZfUlGYK&R6;?X9Zqh@LKQaIyn}a;L5EKT^r*)wBJCP+Je|}JK z{@v+woX{{-dobs7Fst%c{~pj0qxY$$Te?)OX=GrXww4d5n!ZUn@q+y_ zj>_Hhr-}Ew10dGEJ;w?m;>fgz-x8!Fq9PD$^0eF_p|_9wj6rR6%eU&@Y}kMTUn}e! z><7ceLvO!J8)toN<-6gD=hZj-J>GfI=LJMe=#ap=4B@uN`pn=vxTFup8wG^7cX4Pl^kkJ)b~PoW0R%c#W7)(jg102W2Vcwo z>^xAwptYGc@Ma&fgUjQF*~44=>6zEc7v+5`LQZ7Udb4doQj#U)32OOOOR-n_XNsgt zA+!@iw|LbbS!c?CBXZ}T@-GIUZTnrq>eoG7z4l(Iv4f%dk~dGdjuoS<$CQAQ&lbU(zKo<7pCVaKy z(tcUgJ!q+$d6{M>zAiuZr?XaNl{dzxc;L}4&UAEjctZoj+lyniM&oOt79&!&lo5Y7_soUq~0*iV^caTJ{xUAUY zIh2XW?Vhn9!kG=fYi=-*9SqCh)b2TzHc?!3r;8zA;!&P}?qw}@H57QDerht`kGvwi z!=%}1e-?ML!uit}&s>4V*FZ$ryI5uHRTacuRB_QjK@0;cCyK@$xKrjc+Z49c)Gp-X z6E>q25F}_i2Eo0)qO1{(=l3?{%@SfFVL=K{`Cc_xY}e%mZMrMqVPpl6KCs-*KZMFL z(chBg)bL_jJz81=eV&VIK+@x*(zcuh(JoGC@P!-K>_aomM zeIlGB9&|qcIRBL|L(}0= zoueZgp`YaUN@&{9_~*|0W_qcUZ&g6H(^6AYM?b{@E{h$TmmN@fGG!kU)`e=i7~m(H z+8bq92je8Ls6N98vS-?Ug?H;j(2WTFd=|t3_r)%#*9X-LbY4S$EQv`p?#l=Nd)=OATwY)p@XC^Ehlr)9rQx+$rX-W!w7 zIDH+U0Lf0jRsJ*Kzc|)w=2~fsF`Bn3G@<18sbdeP0=%BSsk=Cp;C@Dfff&T%Ehb0f=_i*+$djS0MfLT9!}1@TwO1WisvsFcp#weYw{?8uOtiMnPwUa}Kn#@3j?CqoNBxqnZzUKRT%G+<+>JkLVlCbcM)3qEo zA4a6If3+lJcw&C3MN^j_J#D~{^KK!oBZ=2+`A3%u850Z`(1|~rnlJ+p{3T$?2;ms=4biib5a@U;jw+N}l{Yaut8VF^pBobRNa4|#IECCqh2Hm9 z5H4OH%&d`-%jeE{AiGZuvg+dSv*jJN>{79GVL@JjrCiizBaztNi*i2_;QS7X4-w`- zpQZNrAPlFJ+oPTm4^wQ^alb$}ZVaUFVdAGq-_yOkre?!=r+gB&BZfS?1Hq8fH2fqN z7MN(r#bos)e5g71Z&=}cR!YvU2~&0@UDT~i`hK|5uHykQvMW;itIl|q^|K%Rf1N+l zQy@3t?~MhJz#t1^R;s(-fGi~BVH8%Wpp%CjE+tDVvt!VNI0BJh`YWS- zHTE-}rHQtf65l;@HCQh=Pw*>0)JR_MV9$GK?)D_{T3-CKj*?)77K60&(T1cs@2b*^ zml{z#aro3?4KR-))o6eURsPz6#Sml7`QBw=Jp14Z?7+#Al7jyrVd@_DkxDp4+K)293(%GMny%@i~Im&8m?1$J4@Tv4m&dUqFpsQlv~I z5&MCU!(8kZmga>Ydu2uRqfjP^YtQN=Try8i%fD#3RXQBLaWhUG+E8f^$D%7J{?vMq zHvnEpH+Hu+~_dV?Vc}x z2&lb93*$#DueqTY1AAXLAT>d)$|nDa?{RY^^6J$HzZ>@veybyTg-9@dfE_$$AZ5E) zR~3<}4FP?*^WMi%Duzh?SSFzmg4wDpG7Tba+oWnPi=2N(j-0qrq7a6g!|m!Uv7lw| zD&?g#18{?DsbUr>m}b1LCZ}6f7b0*r3@-t3S2(FNcH&JJI%jM6)?#nu$5FPK6;1TC zD`U+D@{MOb7sw}vdyv>`=jLpH3UVD6tmCBCg6~`w^?sLE`mFK=clhffloowTmzIba zFM4sV^WI2KKcHN@dlNP>$$xwyT6lfr!D1-6pAELOHkR`keyB%;s95#U@#(QaJ}R7V z_IG^ddEC1abEV>>w0<^oeBV;vkQh)A`=zabvR9HyVlk8 zOdtcz&7jufCK+gRC%Jxe{sq2OxUG8MPHP>!96Zu}$4g`3bo{F5HHSI}kIKm>L2CmI zcKh`O08xd-t-c+7v{LyNex2(f7gw6(xC?x}%57fPZh)~r5MK%D?#L#zo9`lb)MQiW zNJV_w6iMhx;^%rt+F7ZpM&$eDi1dc5-;p4d_Zm{vou62yB`Q2H&F_qxt8l(H!m9J$ zF{0UDJF-vwO?%y*b0uE{;83lOH?91y<@W1R8Lparkl&l_zx@Ka<@~fI2iY>`Hb zLOgxrU|e7=gM@3ZTUP;kQ_-IpvCMWMX8i~s_l4hoRJ0Fu& z(})V(j8J)=j*uaA%S|>zD9FxsyI1^;slVC%4GKwlt}AEw7n8gHrw-`& zN;8UgzT?YimH7E};NxjCb+n0jEw&(c9fuXJMF#j4Czrf*_L1G|D~zxuJf`GzJ7O`E zBSSdj-I04&tbKwuMM^)=z_;_d4bT8j?wbkRfwgpcyxWdW3ulo1w$LuSCd!_WHxxdb-_3S)iO$@C$3>>KNQ8}WKqSq0d z(#yIfZ#14sgM}5*PK~O5MTP7HZ+U!H|HhioXAk?Z#ha4mb!26KA#ANw^_QJjylZh6 zdFkx4O|87DE5@~o-B-{T3T!s75+Git%Q1qgRQ8WDoEe9j&sie49rWg!WZ2PRhyCflbIv|K&`}Zs)cdp;fVa zypFdO>FD~$tO|G8kfAjBg9P?k2Gg#I8=*P=U~ze;@U^I=T?3Ym)9vg$aT9h{_(>EA#W$&w>@EERTtt`JwF8U`gn>~2?omf4-0E&r<5*=^*}7*QQ~d; zARUfIk+e1ycVt6W#Jqhv5q97`9gmGqzCX`!$v2W`of!hm=dZ|Gj3A7GWl`ES$GnOI zQ|h*6l4p5aw*9_u3B;FNy2FNdBuV^Q$~xQU5y0&aoDV2|9tEPtKfbeM&m>>kg*UsC z0IpjQq^hBE>$spvH=A`vjWFMw@6idvI!}AUo2Jw~swIW^%%o8Rf<^L{%^cw3Xt^fp^S0XfJbZzG;4e&LhF81-Yf%`V70>g{t%lI^Z} z)+#)uOGh)K=$|;%W5OBQ-yBm9vjMq&?5ejsH{JJ<*AH{CX>rLmN1(9_OIqVbO%R}P z5~ToSg!=x`A?M{myE+2Xx>r_q%jOJ0Rmmq1s7WRn>)&supt&&il_V=Jl0@`-q2!$& zS>CrlPViG_KCv$$IUF;|<-DSQ^c2pwGNmiGrPyaUlc4w&vnvw`-B&A@%&U6VxYhbstM{Zli{jgig3G4m+`(9r48u_(pdSTIiK2vi&^#k6Z86{TEV$k@6TtYU$vD%HQg%Y&oD6h)>e59I60QFgr@1Tj_vi%q@iTrZXo4)2Ma_R>uVC z(4lxd_6~%+G+||+rJHZ6ShJNIH}@IM%~;$66eKc2((h8ozdTnDLKN%Tr{Ut)oT_g~ z5GEoFM~LEaWq>rFtsh)p`a#)aDZza~x3F=q;VxOfq$A30g=h?Ma$|E}lSo+^Uc!s~ zFm#MosupzJ2uw6#Q8~#Q(Dh@FELjF;176Lz%#`*#rp9~c1*uC!gkWU#lYm)u z^`Gup;1^8kDPGG0X@pQm3_^)go}Zmhs~LZ<`J==I@u!+lU>7|Vlf0x2CQB#rQ5%m= z!9t^d3itYpOQ@u0wjZy1`8il|a@VwT4u=zxODCT8y;g$P#gpV#TCpSpo#;Nzthc8;2kVl z;!1ZVwuHUvlZpt84+LSaUOjIC?AB5dq`8sQ~(G9j!x_FV;F`f%=5Yio;PBMO&I z1t~7(HL0{Y@&J6^v(G$EhXBEynHGC`z2D< zoT+sk+?aVTjLNK;Z<(AQpW=N+`VO&t)&3eW8kHE(eE3S5tXWcZObT`cg17&MFBi$fL6h_T*e??96VZ}Ai?tEe#5_zj4%{H)pN2GC%USR)P~EY`nuh+pq~J{> z{$N;n--BOe^F30C#A>bgkDf#ik53c$orBfwZ;2LC&epQLzkl)GrTsqNn6mNFJ=U4$ zJt}Gx$BP*30=@fQUpR0kODo22hsLiz_Zeq%4kHD(DA3+FYMO2hrUY&yFg^zyboS{> zUh(R^Ap^v|Z2K8-B=#Agg4h5o8V)tOYc<~&xUSE~E7jK;7&n{pPz&SNp}y@^7Kty* ze`l*bYB!nSFwdwbaC9MYZb#&vB%`i^D^^ULeD~)Z68giwH;jNccfP*l)u~y(t;OT( z0WwPnf3PbbWx6+e%dVNOO6-`)AiMmP=_lxR^nfgPWE$=?B= zhf&I|nCM%6w_^=UWprrvuMVgAf18YVPWWJxB zASx|2lb`vnN=ZoFclauizpPGn)F>PiugEyHwE*+p@h@PqBQ%NJ=KYLxvppeDiP-XY znV(Y2!uI9zSYd4StiH?H+uvBikBxiV=b5X)WV$U?{*IkDX*Ez->kQ;TXSO$QU0~6{Wy|e}?v>A|^D%y^qyx>9hd`w58xOofCngSnR&;2r zAg!Hz*1hWR6aT>uesVo)NA8LDdF>%Pr;i^^WQ}msNbJdz2!`p7t(MuY?;{nytCrT7 zu{in=l=bor{O8Hy&XOq9T>DFHs`(My^6a{xvyH#N&@~Ws*&zKM_8|K@>s~y)xS`^B zGnCWlSZK&@rsL!FA!S*}AoQt1+XqT^0y0Dud3a8je_k3xKW>JM2ubGBx!_QKzpU${`HKe1Rp4;`nO_B*647k8HV34)KA7 zJ|=|Y1H`+k59gk}H0KmMniJ?+di%JX}Abx;qK{E8t(3!wvhIV2PlsH3!J}obfo9G6Bv0X zMjI+{)%|pB{#LN-6V8TU>e?1qqv)xCBFZ;MgsQt|kv#;|E(M$-}2 zTAm8dB3Jk0?EQ$B?=Mhx zkG9lWB>$*5w%MxL73~Pf3wf8h+3MlQrDEJtkMIW8LSDtclhC=VX0-dGm{(<|rh2<+ zjVOZf-CqzGf=6JbWj&|0`9ntiF_a&oN1CNO`45On*Sbpd#{{6kVwlQ**UtYo?oBtR zvs4J8zms%1vo-x?ol0iS09{EI8^NsptIXKJfIj-a^#7N_Fhp2>=2DfC0&20t_`(n! z`~Fs(%RxkwP~wNkvwTF9ePg+$a@xOeY3i}+O^X2pUcZlU;!Nsa{S@By5tc^-NBGWo zIH^XN2WE*F7}(;!&jNJHiYYWQS5y2W@+H@7O+ruHqYBFb=h7M3^HF%RBMS1{CCH%Z z*S6l)t4@!!Qst&=O6gx4gob>z7{yPnksi6`LKodU)qU6&j;kDHKX;mej3=c^y>7iG z7G;Az$R|EOSxk6L%m0B#i7b9#cc^4?MHgiXfZ+6JUX%M{jq?q>9rM5Abwbl$*VPv; z>}Ia@=go8ZRLs~Sv7;<5xYej%&B3k8J2HQx%S0w`O)#78hw>jK(8lJULbWSRx#V#k z4A*L5%2Vwv%NnCS;6+0IX(76jc{#|N&snJULZDU_Xsq_t^(1nF&c0C-?FSOwi>Fx`U7*Jsz*Q^Vk>pMC-2T+-1G%u^UIRoujn&fS^hNDGf_pZhoT{AZa9CBlu z7;y@{Yl}7q_I-uNX+)#p(_ihE&#so((>x$s51o*j8O+Kmo0^G41Tn6Ak8V#svU4IH zF3(c&J1d2$Y@O}fuWG$GzZImkbr#3^qkBWZZ{Ygh&Imw@yLcef;%P?Q0IyFu2NrWc z?*a;6izgsu*A9o*q_4O3+|W*o0|n$udVbqk*>8OzG-qRcqMN8gqKxcsX(tuJ30j{lna6nlEn0BabAI^o*R47Gmu zVYw>$F+u9GSPxJA&cd>%7t?4bNXmrmurm|96$%15kKVEZVW)rmVq*Y56 zsSw~F4S3o75oL4vL3TwQV^$nr1C>CdB;eCSmQ$7J=ZaAX;tx@(y+-(gUhULXyg0Zd z&5sB^UcBiNEFzNEn=vZlV{F~G-^>05jdoTM9#40s_OC?|+&* zbl*f$A~X0Ck=1DS&tv6^rN!G|1mb-}3strn)P-{Aii)fnbiC^*0t639f&VWy8_Wm# zV-_#1#XAxcEL>7*62SO@!}{%qOU`K`hHfL zH8%Dtz3F_F?byWKk9gJ#HoYTvCQlx{_>#eM@Ytyt--i#ru{7|$a3IfJuSg=N#>GRK zZo~2J{MNGh>7bqUxe*a4TmtFrG~{I!P}0g8wtc~QH*)PohJSZKSMp60*V??1+@j14 zP4(AUkFG&HbY11QR$j33{Dvj18Cue{&Dbj(jKj8zN7F<@I8x65?Nej^pKL2RrnH3^F;_$99+^Dz$+ E17&ypf&c&j literal 0 HcmV?d00001 From 7151fb39d5625b6f80a29a82d138e2936d10093b Mon Sep 17 00:00:00 2001 From: Dusan Andric Date: Thu, 3 Jun 2021 12:25:04 -0400 Subject: [PATCH 3/8] md5 conversion from base64 to hex for azure blob storage --- .../repository/azure/AzureDownloadService.java | 18 ++++++++++++++++-- .../azure/AzureDownloadServiceTest.java | 17 +++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java diff --git a/score-server/src/main/java/bio/overture/score/server/repository/azure/AzureDownloadService.java b/score-server/src/main/java/bio/overture/score/server/repository/azure/AzureDownloadService.java index 17e235d0..c37fbc32 100644 --- a/score-server/src/main/java/bio/overture/score/server/repository/azure/AzureDownloadService.java +++ b/score-server/src/main/java/bio/overture/score/server/repository/azure/AzureDownloadService.java @@ -31,6 +31,7 @@ import com.microsoft.azure.storage.blob.CloudBlobContainer; import com.microsoft.azure.storage.blob.CloudBlockBlob; import lombok.Setter; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.springframework.beans.factory.annotation.Autowired; @@ -40,6 +41,7 @@ import org.springframework.stereotype.Service; import java.net.URISyntaxException; +import java.util.Base64; import java.util.List; import static com.google.common.base.Preconditions.checkArgument; @@ -99,8 +101,8 @@ public ObjectSpecification download(String objectId, long offset, long length, b } fillPartUrls(objectId, parts); - return new ObjectSpecification(objectId, objectId, objectId, parts, rangeLength, blob.getProperties() - .getContentMD5(), false); + val md5 = base64ToHexMD5(blob.getProperties().getContentMD5()); + return new ObjectSpecification(objectId, objectId, objectId, parts, rangeLength, md5, false); } catch (StorageException e) { log.error("Failed to download objectId: {}, offset: {}, length: {}, forExternalUse: {}: {} ", objectId, offset, length, forExternalUse, e); @@ -146,4 +148,16 @@ public String getSentinelObject() { null); return result; } + + @SneakyThrows + static String base64ToHexMD5(String content) { + val bytes = Base64.getDecoder().decode(content); + val output = new StringBuilder(); + for (val b : bytes) { + output.append(String.format("%02x", b)); + } + + log.trace("Converted MD5 from {} to {}", content, output); + return output.toString(); + } } diff --git a/score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java b/score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java new file mode 100644 index 00000000..8cbfe051 --- /dev/null +++ b/score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java @@ -0,0 +1,17 @@ +package bio.overture.score.server.repository.azure; + +import junit.framework.TestCase; +import lombok.val; +import org.junit.Test; + +public class AzureDownloadServiceTest extends TestCase { + + @Test + public void testContentMD5Conversion() { + val contentMD5 = "iC9XnDziOQvKwyeOSUfKIg=="; + val converted = AzureDownloadService.base64ToHexMD5(contentMD5); + + assertEquals("882f579c3ce2390bcac3278e4947ca22", converted); + } + +} \ No newline at end of file From d3057349c782d9a6d05a41dd628eb3b1cade53f1 Mon Sep 17 00:00:00 2001 From: Dusan Andric Date: Thu, 3 Jun 2021 12:31:47 -0400 Subject: [PATCH 4/8] missing newline from test --- .../score/server/repository/azure/AzureDownloadServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java b/score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java index 8cbfe051..4d91d278 100644 --- a/score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java +++ b/score-server/src/test/java/bio/overture/score/server/repository/azure/AzureDownloadServiceTest.java @@ -14,4 +14,4 @@ public void testContentMD5Conversion() { assertEquals("882f579c3ce2390bcac3278e4947ca22", converted); } -} \ No newline at end of file +} From 63b9378e980efbfb5550529ebfb8457d6133d280 Mon Sep 17 00:00:00 2001 From: jaserud Date: Fri, 18 Jun 2021 15:16:02 -0400 Subject: [PATCH 5/8] make upload state parent directory configurable --- .../score/client/upload/UploadStateStore.java | 47 ++++++++++--------- .../client/upload/s3/S3UploadService.java | 26 +++++++--- .../src/main/resources/application.yml | 9 ++-- .../client/upload/UploadStateStoreTests.java | 6 +-- 4 files changed, 53 insertions(+), 35 deletions(-) diff --git a/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java b/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java index b2ebdc87..2e17b4dc 100644 --- a/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java +++ b/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java @@ -41,34 +41,28 @@ public class UploadStateStore extends TransferState { * /path/to/./uploadid upload state file */ - /* - * We are using the directory that the upload file is currently located in. Requires read/write access to directory - * Otherwise, we need to have user specify a working directory for upload - which is a bit too counter-intuitive. - */ @SneakyThrows - public static File getContainingDir(@NonNull final File uploadFile) { - val parentPath = Optional.ofNullable(uploadFile.getParent()); - if (parentPath.isPresent()) { - return new File(parentPath.get()); - } - return null; + public static String getContainingDir(@NonNull final File uploadFile) { + return uploadFile.getParent(); + } + + public static void create(@NonNull String uploadStateParentDirPath, @NonNull ObjectSpecification spec, boolean force) throws NotRetryableException { + create(uploadStateParentDirPath, spec); } /** * Write upload-id of current upload into state directory (hidden directory next to file being uploaded) - * @param uploadFile - File we want to upload. Used to determine path to create temporary upload id directory + * @param uploadStateParentDirPath - Path to create temporary upload id directory * @param spec - * @param force */ - public static void create(@NonNull File uploadFile, @NonNull ObjectSpecification spec, boolean force) { - val filePath = getContainingDir(uploadFile); - + public static void create(@NonNull String uploadStateParentDirPath, @NonNull ObjectSpecification spec) throws NotRetryableException { try { - val uploadStateDir = getObjectStateDir(filePath, spec.getObjectId()); + val objectStatePath = getObjectStatePath(uploadStateParentDirPath, spec.getObjectId()); + val objectStateDir = new File(objectStatePath); - removeDir(uploadStateDir, true); + removeDir(objectStateDir, true); - val uploadIdFile = new File(uploadStateDir, getStateName()); + val uploadIdFile = new File(objectStateDir, getStateName()); try (PrintWriter out = new PrintWriter(uploadIdFile, StandardCharsets.UTF_8.name())) { out.println(spec.getUploadId()); } @@ -82,10 +76,10 @@ protected static String getStateName() { return "uploadId"; } - public static Optional fetchUploadId(@NonNull File uploadFile, @NonNull String objectId) { - Optional result = Optional.ofNullable(null); - val uploadStateDir = getObjectStateDir(getContainingDir(uploadFile), objectId); - val uploadIdFile = new File(uploadStateDir, getStateName()); + public static Optional fetchUploadId(@NonNull String uploadStateParentDirPath, @NonNull String objectId) { + Optional result = Optional.empty(); + val objectStatePath = getObjectStatePath(uploadStateParentDirPath, objectId); + val uploadIdFile = new File(objectStatePath, getStateName()); if (uploadIdFile.exists()) { try (val reader = @@ -103,4 +97,13 @@ public static Optional fetchUploadId(@NonNull File uploadFile, @NonNull } return result; } + + public static void close(@NonNull String uploadStateParentDirPath, @NonNull String objectId) throws IOException { + val dirToDelete = new File(getObjectStatePath(uploadStateParentDirPath, objectId)); + deleteDirectoryIfExist(dirToDelete); + } + + private static String getObjectStatePath(@NonNull String uploadStateParentDirPath, @NonNull String objectId) { + return uploadStateParentDirPath + "/." + objectId; + } } diff --git a/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java b/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java index f5e11361..e9d4744d 100644 --- a/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java +++ b/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java @@ -52,6 +52,8 @@ public class S3UploadService implements UploadService { private boolean quiet; @Value("${storage.retryNumber}") private int retryNumber; + @Value("${client.uploadStateParentDirPath}") + private String uploadStateParentDirPath; /** * Dependencies. @@ -104,16 +106,18 @@ private void startUpload(File file, String objectId, String md5, boolean overwri ObjectSpecification spec = null; try { spec = storageService.initiateUpload(objectId, file.length(), overwrite, md5); + + // Delete if already present + if (overwrite) { + UploadStateStore.create(getUploadStateParentDirPath(file), spec, false); + } } catch (NotRetryableException e) { // A NotRetryable exception during initiateUpload should just end whole process // a bit of a sleazy hack. Should only be thrown when the Metadata service informs us the supplied // object id was never registered/does not exist in Metadata repo - throw new NotResumableException(e); - } - // Delete if already present - if (overwrite) { - UploadStateStore.create(file, spec, false); + // being unable to create file for upload state should also end whole process + throw new NotResumableException(e); } val progress = new Progress(terminal, quiet, spec.getParts().size(), 0); @@ -144,7 +148,7 @@ private UploadProgress checkProgress(File uploadFile, String objectId) { // See if there is already an upload in progress for this object id. Fetch upload id and send if present. If // missing, send null - val uploadId = UploadStateStore.fetchUploadId(uploadFile, objectId); + val uploadId = UploadStateStore.fetchUploadId(getUploadStateParentDirPath(uploadFile), objectId); val progress = storageService.getProgress(objectId, uploadFile.length()); // Compare upload id's @@ -231,7 +235,15 @@ public boolean isObjectExist(String objectId) throws IOException { } private void cleanupState(File uploadFile, String objectId) throws IOException { - UploadStateStore.close(UploadStateStore.getContainingDir(uploadFile), objectId); + UploadStateStore.close(getUploadStateParentDirPath(uploadFile), objectId); } + private String getUploadStateParentDirPath(File uploadFile) { + if (!uploadStateParentDirPath.equals("")) { + return uploadStateParentDirPath; + } else { + val path = UploadStateStore.getContainingDir(uploadFile); + return path; + } + } } diff --git a/score-client/src/main/resources/application.yml b/score-client/src/main/resources/application.yml index 45789217..24ae3317 100644 --- a/score-client/src/main/resources/application.yml +++ b/score-client/src/main/resources/application.yml @@ -29,6 +29,9 @@ client: accessToken: "${accessToken:}" # Alias for backwards compatibility connectTimeoutSeconds: 60 readTimeoutSeconds: 60 + # path of dir with WRITE access for score client upload state files + # if empty uses parent dir of current file to upload + uploadStateParentDirPath: "" ssl: custom: false @@ -51,15 +54,15 @@ transport: storage: # default for backwards compatibility - url: https://virginia.cloud.icgc.org + url: http://localhost:8087 retryNumber: 10 retryTimeout: 1 metadata: # default for backwards compatibility - url: https://virginia.song.icgc.org + url: http://localhost:8089 ssl: - enabled: true + enabled: false portal: url: https://dcc.icgc.org diff --git a/score-client/src/test/java/bio/overture/score/client/upload/UploadStateStoreTests.java b/score-client/src/test/java/bio/overture/score/client/upload/UploadStateStoreTests.java index 07f7fbef..8485cf9d 100644 --- a/score-client/src/test/java/bio/overture/score/client/upload/UploadStateStoreTests.java +++ b/score-client/src/test/java/bio/overture/score/client/upload/UploadStateStoreTests.java @@ -11,8 +11,8 @@ public class UploadStateStoreTests { - File getTestResourceRoot() { - return new File(getClass().getClassLoader().getResource("fixtures/upload/placeholder-upload-file.txt").getFile()); + String getTestResourceRoot() { + return getClass().getClassLoader().getResource("fixtures/upload/").getPath(); } @Test @@ -31,7 +31,7 @@ public void test_fetch_upload_id_finds_improperly_formatted_file() throws IOExce @Test public void test_upload_state_store_containing_dir() throws IOException { - val pathString = getTestResourceRoot().getCanonicalPath(); + val pathString = getTestResourceRoot(); val testFile = new File(pathString); val resultFile = UploadStateStore.getContainingDir(testFile); assertThat(resultFile).isNotNull(); From b0d3d7fee5160cbc9973790dd5c40a144b1f4bd8 Mon Sep 17 00:00:00 2001 From: jaserud Date: Fri, 18 Jun 2021 15:20:28 -0400 Subject: [PATCH 6/8] revert local configs --- .../bio/overture/score/client/upload/UploadStateStore.java | 4 ---- .../overture/score/client/upload/s3/S3UploadService.java | 2 +- score-client/src/main/resources/application.yml | 6 +++--- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java b/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java index 2e17b4dc..32cd0b6b 100644 --- a/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java +++ b/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java @@ -46,10 +46,6 @@ public static String getContainingDir(@NonNull final File uploadFile) { return uploadFile.getParent(); } - public static void create(@NonNull String uploadStateParentDirPath, @NonNull ObjectSpecification spec, boolean force) throws NotRetryableException { - create(uploadStateParentDirPath, spec); - } - /** * Write upload-id of current upload into state directory (hidden directory next to file being uploaded) * @param uploadStateParentDirPath - Path to create temporary upload id directory diff --git a/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java b/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java index e9d4744d..f23be07a 100644 --- a/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java +++ b/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java @@ -109,7 +109,7 @@ private void startUpload(File file, String objectId, String md5, boolean overwri // Delete if already present if (overwrite) { - UploadStateStore.create(getUploadStateParentDirPath(file), spec, false); + UploadStateStore.create(getUploadStateParentDirPath(file), spec); } } catch (NotRetryableException e) { // A NotRetryable exception during initiateUpload should just end whole process diff --git a/score-client/src/main/resources/application.yml b/score-client/src/main/resources/application.yml index 24ae3317..2070ac89 100644 --- a/score-client/src/main/resources/application.yml +++ b/score-client/src/main/resources/application.yml @@ -54,15 +54,15 @@ transport: storage: # default for backwards compatibility - url: http://localhost:8087 + url: https://virginia.cloud.icgc.org retryNumber: 10 retryTimeout: 1 metadata: # default for backwards compatibility - url: http://localhost:8089 + url: https://virginia.song.icgc.org ssl: - enabled: false + enabled: true portal: url: https://dcc.icgc.org From 0c40a4512a3c6673188c2228e7df898f4cc2f985 Mon Sep 17 00:00:00 2001 From: jaserud Date: Mon, 21 Jun 2021 11:30:44 -0400 Subject: [PATCH 7/8] less verbose name, and use isEmpty --- .../score/client/upload/UploadStateStore.java | 18 +++++++++--------- .../client/upload/s3/S3UploadService.java | 19 +++++++++---------- .../src/main/resources/application.yml | 2 +- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java b/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java index 32cd0b6b..53c78f16 100644 --- a/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java +++ b/score-client/src/main/java/bio/overture/score/client/upload/UploadStateStore.java @@ -48,12 +48,12 @@ public static String getContainingDir(@NonNull final File uploadFile) { /** * Write upload-id of current upload into state directory (hidden directory next to file being uploaded) - * @param uploadStateParentDirPath - Path to create temporary upload id directory + * @param uploadStateDir - Path to create temporary upload id directory * @param spec */ - public static void create(@NonNull String uploadStateParentDirPath, @NonNull ObjectSpecification spec) throws NotRetryableException { + public static void create(@NonNull String uploadStateDir, @NonNull ObjectSpecification spec) throws NotRetryableException { try { - val objectStatePath = getObjectStatePath(uploadStateParentDirPath, spec.getObjectId()); + val objectStatePath = getObjectStatePath(uploadStateDir, spec.getObjectId()); val objectStateDir = new File(objectStatePath); removeDir(objectStateDir, true); @@ -72,9 +72,9 @@ protected static String getStateName() { return "uploadId"; } - public static Optional fetchUploadId(@NonNull String uploadStateParentDirPath, @NonNull String objectId) { + public static Optional fetchUploadId(@NonNull String uploadStateDir, @NonNull String objectId) { Optional result = Optional.empty(); - val objectStatePath = getObjectStatePath(uploadStateParentDirPath, objectId); + val objectStatePath = getObjectStatePath(uploadStateDir, objectId); val uploadIdFile = new File(objectStatePath, getStateName()); if (uploadIdFile.exists()) { @@ -94,12 +94,12 @@ public static Optional fetchUploadId(@NonNull String uploadStateParentDi return result; } - public static void close(@NonNull String uploadStateParentDirPath, @NonNull String objectId) throws IOException { - val dirToDelete = new File(getObjectStatePath(uploadStateParentDirPath, objectId)); + public static void close(@NonNull String uploadStateDir, @NonNull String objectId) throws IOException { + val dirToDelete = new File(getObjectStatePath(uploadStateDir, objectId)); deleteDirectoryIfExist(dirToDelete); } - private static String getObjectStatePath(@NonNull String uploadStateParentDirPath, @NonNull String objectId) { - return uploadStateParentDirPath + "/." + objectId; + private static String getObjectStatePath(@NonNull String uploadStateDir, @NonNull String objectId) { + return uploadStateDir + "/." + objectId; } } diff --git a/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java b/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java index f23be07a..61ac941b 100644 --- a/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java +++ b/score-client/src/main/java/bio/overture/score/client/upload/s3/S3UploadService.java @@ -52,8 +52,8 @@ public class S3UploadService implements UploadService { private boolean quiet; @Value("${storage.retryNumber}") private int retryNumber; - @Value("${client.uploadStateParentDirPath}") - private String uploadStateParentDirPath; + @Value("${client.uploadStateDir}") + private String uploadStateDir; /** * Dependencies. @@ -109,7 +109,7 @@ private void startUpload(File file, String objectId, String md5, boolean overwri // Delete if already present if (overwrite) { - UploadStateStore.create(getUploadStateParentDirPath(file), spec); + UploadStateStore.create(getUploadStateDir(file), spec); } } catch (NotRetryableException e) { // A NotRetryable exception during initiateUpload should just end whole process @@ -148,7 +148,7 @@ private UploadProgress checkProgress(File uploadFile, String objectId) { // See if there is already an upload in progress for this object id. Fetch upload id and send if present. If // missing, send null - val uploadId = UploadStateStore.fetchUploadId(getUploadStateParentDirPath(uploadFile), objectId); + val uploadId = UploadStateStore.fetchUploadId(getUploadStateDir(uploadFile), objectId); val progress = storageService.getProgress(objectId, uploadFile.length()); // Compare upload id's @@ -235,15 +235,14 @@ public boolean isObjectExist(String objectId) throws IOException { } private void cleanupState(File uploadFile, String objectId) throws IOException { - UploadStateStore.close(getUploadStateParentDirPath(uploadFile), objectId); + UploadStateStore.close(getUploadStateDir(uploadFile), objectId); } - private String getUploadStateParentDirPath(File uploadFile) { - if (!uploadStateParentDirPath.equals("")) { - return uploadStateParentDirPath; + private String getUploadStateDir(File uploadFile) { + if (!uploadStateDir.isEmpty()) { + return uploadStateDir; } else { - val path = UploadStateStore.getContainingDir(uploadFile); - return path; + return UploadStateStore.getContainingDir(uploadFile); } } } diff --git a/score-client/src/main/resources/application.yml b/score-client/src/main/resources/application.yml index 2070ac89..40d955ba 100644 --- a/score-client/src/main/resources/application.yml +++ b/score-client/src/main/resources/application.yml @@ -31,7 +31,7 @@ client: readTimeoutSeconds: 60 # path of dir with WRITE access for score client upload state files # if empty uses parent dir of current file to upload - uploadStateParentDirPath: "" + uploadStateDir: "" ssl: custom: false From 5acda969e1b2311e425f2fe2da9770716398cc9a Mon Sep 17 00:00:00 2001 From: jaserud Date: Tue, 22 Jun 2021 16:20:04 -0400 Subject: [PATCH 8/8] update to 5.4.0 --- pom.xml | 2 +- score-client/pom.xml | 2 +- score-core/pom.xml | 2 +- score-fs/pom.xml | 2 +- score-server/pom.xml | 2 +- score-test/pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 882beeb5..ca0e7485 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.4.0-SNAPSHOT + 5.4.0 pom ${project.artifactId} diff --git a/score-client/pom.xml b/score-client/pom.xml index 4de048ab..1ac34d59 100644 --- a/score-client/pom.xml +++ b/score-client/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.4.0-SNAPSHOT + 5.4.0 ../pom.xml diff --git a/score-core/pom.xml b/score-core/pom.xml index 4a188eff..536e4b86 100644 --- a/score-core/pom.xml +++ b/score-core/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.4.0-SNAPSHOT + 5.4.0 ../pom.xml diff --git a/score-fs/pom.xml b/score-fs/pom.xml index 66c4b9b3..ba1a32b2 100644 --- a/score-fs/pom.xml +++ b/score-fs/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.4.0-SNAPSHOT + 5.4.0 ../pom.xml diff --git a/score-server/pom.xml b/score-server/pom.xml index 52763f2c..7e3fd5d3 100644 --- a/score-server/pom.xml +++ b/score-server/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.4.0-SNAPSHOT + 5.4.0 ../pom.xml diff --git a/score-test/pom.xml b/score-test/pom.xml index 66b333fe..5d6ad601 100644 --- a/score-test/pom.xml +++ b/score-test/pom.xml @@ -21,7 +21,7 @@ ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF S bio.overture score - 5.4.0-SNAPSHOT + 5.4.0 ../pom.xml