From 73fd7024bd4f8f9f684ad1df1fcfba76c933a728 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett <117409426+maxinelasp@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:53:13 -0700 Subject: [PATCH] MAG CDF updates (#349) * Updating mag CDF generation to generate two files * PR updates, naming changes --- imap_processing/cli.py | 23 ++-- ..._mag_l0_test_20231212_20231212_v00-01.pkts | Bin 0 -> 28500 bytes imap_processing/mag/l0/decom_mag.py | 112 +++++++++++++----- imap_processing/mag/l1a/mag_l1a.py | 23 +++- imap_processing/mag/mag_cdf_attrs.py | 4 + imap_processing/tests/mag/test_mag_decom.py | 46 ++++++- 6 files changed, 163 insertions(+), 45 deletions(-) create mode 100644 imap_processing/data/imap/mag/l0/2023/12/imap_mag_l0_test_20231212_20231212_v00-01.pkts diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 8f40fcc87..9d465f746 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -11,6 +11,7 @@ """ import argparse +import logging import sys from abc import ABC, abstractmethod from json import loads @@ -35,6 +36,8 @@ from imap_processing.swe.l1a.swe_l1a import swe_l1a from imap_processing.swe.l1b.swe_l1b import swe_l1b +logger = logging.getLogger(__name__) + def _parse_args(): """Parse the command line arguments. @@ -313,17 +316,23 @@ def process(self): f"Unexpected dependencies found for MAG L1A:" f"{file_paths}. Expected only one dependency." ) - filename = imap_data_access.ScienceFilePath.generate_from_inputs( - "mag", "l1a", "raw", self.start_date, self.end_date, self.version - ) - mag_l1a(file_paths[0], filename.construct_path()) - print(f"Generated file: {filename.construct_path()}") + filename_norm = imap_data_access.ScienceFilePath.generate_from_inputs( + "mag", "l1a", "raw-norm", self.start_date, self.end_date, self.version + ).construct_path() + filename_burst = imap_data_access.ScienceFilePath.generate_from_inputs( + "mag", "l1a", "raw-burst", self.start_date, self.end_date, self.version + ).construct_path() + mag_l1a(file_paths[0], filename_norm, filename_burst) if self.upload_to_sdc: - print(f"Uploading file from: {filename.construct_path()}") # TODO: figure out data_dir, because now this fails. # Should switch to using IMAP_DATA_DIR env var. - imap_data_access.upload(filename.construct_path()) + if filename_norm.exists(): + logging.info(f"Uploading file: {filename_norm}") + imap_data_access.upload(filename_norm) + if filename_burst.exists(): + logging.info(f"Uploading file: {filename_burst}") + imap_data_access.upload(filename_burst) class Swapi(ProcessInstrument): diff --git a/imap_processing/data/imap/mag/l0/2023/12/imap_mag_l0_test_20231212_20231212_v00-01.pkts b/imap_processing/data/imap/mag/l0/2023/12/imap_mag_l0_test_20231212_20231212_v00-01.pkts new file mode 100644 index 0000000000000000000000000000000000000000..36eb71aff2a1c1164add60853efd4b5f52b191e3 GIT binary patch literal 28500 zcmciLbyQRD1Nd>JV|2ri?%L=W(mlFibi;s+G%yel5R`5#$^dl(5d)PaARrwQ2A~+I zn_`0c+wSl4J)HC7@8AB}@%YdCyu9~r_dfS|9zo!Nefw4<`DTvti?D|)(*A(jp8da{ z`v=(f@$MUVX_M=~L@~B}1;rzIWsOgwnVCy?wZK6GYkB7irBNpQ>*%2!ULO;Wv@6W@ zypdqyz#Br-pb^G|vVj|K<}ET+%y4IJ=dG328hCGWjok}kI`BbYPLsFa^k?Q*Ml|+; zzytdMW8Qzp=^xu)B>Cu{%-y^r(sctL^X{-CQD)_zB9=#aDQ2VDP0Ry)`wr|Ipb!WS zrzx;->g6>ApCYXJum$rNpTU8|fi32V!pjhI$QJe8CLh8ab!3BikuTxE^uQ0BmB9y? zgZw|OD5ZSm2W@jo7}1~`nE+$Su8|5DsmvRE7i7!_e&@X(oTVImxt;q1%ty2kKd#5T z#WyX}G4Pl9Cp!gYf%!Z6r=Rbq#oH6}EKK|yvi!LHwycg^7|UEHc@{Xore%Nb2^Kzn znCxL3OTM6^BE`y+HH~YNKiaB1Pk|8)x{(<$R_z)ofRVxi=10q};n;~XMHVP)IgW0D z9R7Oi==>2DHU17cIF6e|b0`pFvmeAH4(1=V(LbHaV#vQJ*NfxLH|5BH*i7=-VzAKVU?IZe#(BO}oa=fblbnr+}?|5l)1~ z$MGt~4lS~(u`GbJ!<0m^gbEy%|AG^@jVK~O?5!kpb#nwR*`GYi&XOc>Lm?O^m7g-S zf^m=opE3an%sBX*>ts13u&O}99UvYZ`ayA^%1l{w33529mU}azK{v7j#@1cqCScrT zDG(G>)W*r%7IO-s9B~SQ_Ao)1W zq|$zzDzUZL0pc8?*6ocHyy|?Z#)jpFV6RdVPLl;w9D#CyYJLga5nOi3sQti*2HnU8 z7(4eG?+dg{^Wt^svt*dyW z5zCB_xAGO7Aco{14*+wZ!L^ z3X6M0Um9oS6joMs#o4feo=8wU_uC2>T{ zA~#KU9VrGu4BRVZ9Tzj#K8njIm?~*Ug+yeY`T!D(4yk^i!MZG#roE2KCcPMWObLY^ z!c&{Y>O&o{3x3d4Z0Bo7#DYq|2tQ|i!JH& z;f@#lcG<#&O`eGSJ1V{}95f)q#v;xSk;CPZ+2OpD@XM!qSdrq!;V*FnHUV)v$QfJ# zn{cTbDgs^bm3v1V8G#|hu^kXU0@=b9*~^XEU?M$>Z3LUeFGc2)_*faxpc{Dr+lZ?ru|uDL0Z5i>Dzg%J(9 zkryz&-!;Ag&eb<;;S#xe%(!}T)aW247TrM5E0wq!+cxIHmMn2cFB#X&h8lfEiL*M- zYzmfGjyv@9E87u?4|;RBi>Hr+b|CQrm%J_eCHdnWC+gUWBxUt2aV_K$S5Zv-tJYQr zxTI|Yc*=qi4Z4vJFn;-ejT>w=l0N#^a2;&*t`JHBw&Sboresb6WqOURMY7mH2zSlC z{j3WrG55NjPmUxekvLnzc1!Y+K@P6#boW>cCMn`ZUNBg4J*j$Lhpk`oi@|FghHZf7 zBm@b?Oh9RQooJ9;RYP`KV_z% zJaOKqy1=riqE|xf++ZE!P#j^9kM{ryEk~>>hJk(10jquN65wd#A>1H4*v%M&rVg%^ zje?8PG}pt}mBIBUP}~y-wR7H(G~9?@EfPGCzV8h)BN}w0AYkO$V`SF@kC`;##tIG3 zC8N?So~AeNfVa|THagiYrI}1wa1(_O5=rt`S@!cg9RRF=R(6lPSqJPW?r#E1snCBWD~Ez{Zp@*IRU2aI0mvaDwT!*&z(LqC+&Vg>%ZzTqCqzb z14hYRW?6|me{W!xN7c!``~cHJqUAj|E9tM z;=t%p*!L6ceFv7!@8Lcb-Y?699G(1}{(Mu0C5Qi~0_{0l;GhDIIyhKXhRVTyA-pb? zG0s{4HNuDn-6#SW<@XvVW$X`@;=ZxZj9;T1%l*#036?<~yS<&t{#@qh!SA@Ag{yo- z$Z^kK-e0<8E*&rWBgXzg=9WbmZhP>v`*X|*xt-QuaG9ACh`$fmf61&{4CDT>@3{Y@ zoS^<&`?o9$Itkqu#m;~R-6#qe)pw1nz`450!6qwVsf%ZFV^FPg$|zIffJ_94M#KIA?^qWUrpO#_G+X zCEI6}iU&DD`6E$z);z+JFxln2V{F@uXwZ#ffKhj^(M)#JY5~tzbkHLQlP}00ExRno zpYO_Xk;73=&e|F;IOI}Z2gzR*x~+ngb1abKa^Ucl^R@28i*f{%V^IZoF&1r*Tu#9^ z?hhQ1a%XIW@e+=-dmGB>N=aM&W;x91VV)`u1~lkKalmN0*O)5z*ycDMJe1D=22vO) zU2O)FTQ6+jGvdgR`)ad+m*qI=xersME4ywrDlcA?%)i8OMqbs{6R+TS)>D#F%%rIA zh?Mszwi3+asFM$|y^mKeYOF9o6;G*Txb4X2pAisN;b1_6Zj=Cw)_aX@@|AXqc(tJ` zlkS)^-ReZ&X8F5kUW<%#VCBhnXYpE&y#k4ll2h8k2u=CTl6zw59FOF;?S9}PMRB=dN}tfGCQ*2tvFc(IUwa}Rx^41imK(;!#t(>Zgmz}Kf&chm&{;0}Tm)mtYibGXO$}XG+igQj~csEYtDKw@E@6K`o zthiJ4P31Eq8gwHVFb3^4S}Somi{L$r?FCCA)sS47yPIn~>=M#9JVQ;|> zR86FBTPIv8y5_uw8E1%6hVxr|0B5+@0HszpU>&=xR9~B{`GPZ1>6(idKG-R_atTt$ zgwVg6qcl-xrCrF$fCk+t4HzSLjkMcW+WU?b&TOR>mk0RJ;v-YvFmeh=L6;QK*aS|t4Mbb?lpXM|?U4?0$O1FK}tYUed z)s&BOTE*S%0zPwiUg##|d`njKM@^NC^UuusI2q8O8)X1v+Fs*Jl{|NL{9(>@p&`_T zw8QIPM^&y~xM3d5`AOxjI~t$kMD<>!T(mx>_#2_}@?x$9Gv~I-NB2eiiQ>Oie;~9m z^UQras=}8%Ew6BKsLFZR;7>i_o)N=b`jSUtFI9ECtYq!Z$$$pkC<_>m>@jkQsQP)_ zz@O$4pMg>?;|oQ2HC1yi|FZeYC9hiIDT*&Xqbv+ZwN#$*74BEXv<%zTaY0lEJWt|F zpXm9-Vp=25wo4&Y*IUm!Sa6xEZh5}LmvdQG=R(?a%fHEOs)5_kj%$o)(2a6{@zkzy z3Ai4YxSZA0yu9&M&aTx>sCL0>JJnJ(k9Irf5-xwWP_KS`?U|sNUP}9FU4af*Ex$tq zuEQ0rR^_dXZ{&&>o`7_8H_jS#som{(=Q_`YRvYv#!=HD~^m&iDa_WK-46e3$<)Qmg zMl|R~dBAvPuknQ1j`uJ8r8B2|StwV%F2`6et1Diu@KoR`Rj2*D<6EDU)ySZ($+z`7 z<){Z-i|`ucYEX~$8O2}WYMwR0TxYt<>H$(Oy>8%>%GIIX=xcz#?tE?5n{s`s)7P&{ zeV~)mPl$^F4Z2YQFjnq0_Nb5hUc}#c(kGIF>TJ1L69QA;>Rj=Ez=hLb@#DZ_xrn|+ zn65PJ>!@W7)vg0Ut&khmy#|R$jqn?}!R%a18p(c3_`WmGYaXI* z%HJ(OgEXpd`XV~H7|@^_6#--8uJIXgu0G>>tI^_bi@*Qm!|Xie=9dSg>}HLjTWX=+ zT;DY2{cqwQIsX#*0=b2MEOH#Cv2*KJ*bih$e2@ z>*Ro3JaI(GPnm*^B=wdcHRG|Dqih(_pc|C{W6Q2_0WdCbOKWBZyvGl5%hp<xYaeU2m0VgT(stbF+GB#cJ-y26FqjZW!y%Z&jTOgpN^P`W_iGv7{+O`Dhz8xL3>dHPHF{||29@JyU3~qn zQ+iL$DRm*Vki8F+PH=~59Si!6Unq&JB|+|ZJc%j&YAl~xy$sDbpWglx@&?;=Vm~IZd3t`-Mhwl;5g26 zpVp?0G2&NC&WH&^?oO@wlKZtS?{TJyaaU`5ATHrwkJR~VWA3%QX&-}YXWUy!f6RSZ zJ3oXI|Bk!O-;r`Z?cKNOP3^Ax*vu&Io7(q6((s!u*t#gx1M80li#gga9~5MBGonE^ zsshIQyGGjl9NN9yEcXNLPa(_r&n1uMk74@dDFv^PBxuL;lFZ^iq%8< zzkHqD=+bd~sCLAgdsfFk^cMcR%QLZV)I%?U+Xp?k0bx|Zs+K{eSGP8hb4t43dwR$T#9y5}LPAKrght&LE@DV}!eq5(XaFhZ%s3+&bfX4foZ4#)gvLcaC5XX8#P37M%;KUDO=v0E zq%<283vG%rB1o1diqD|P(^9d9qtJmto^w*54CqAEC4%&5cEBgfVC#Vna|HC~;HxqM zC|8#?nu{O@%5UI=45iC`x7pFv9Ku$_ff&%B8#Mvr;$Gu9U6bf^f+D=)nF40mTgl$J zR5yIMpo$OFq?;1GLQpBa@XVa@L_xLIQ&YG4Nnmvk=&EjOj6FdE)Flysdh%6c&c9!G zXhf~nA9P1|AqGRxhTjWJ!;BDgltU4^J0pMU{(%_Kpc}OS>AEOMyqtQqBr&QM<*LvK@)mbv3Ugj(dmXOsIe%6J4rcu@nh{xE}#{?tk_M0G3ezp z9ObEv30oRiulZ?a^Jma|z0Np4f?4S&iDk&R5X|pTm)^v<(}g-lH0VZc!1#XGH~|%%5!Qv-Pf z^_}7?33jd`jT(^2JbS~^9DURzuCRg z2QYrwW8~4)?@ovzxQyv6hEb;V;ByUK`U}$u9n(A}`Wp%31b0_+$s?#4W)IOzFaywx z`PHL54hG;vV}jRN=b&oL%(QoG$FhOtEYEcr9v=gb#LEQVG5^L}kl9whJ2#L9hh|@O z4l<%aH$nj8?_J|Ka2&^ZA`A+WxCwzg(T&5Xxx65@+aQCkx!xN|JShhElQIYhSM=f< zW!^i){(iH;%lU#^d^|@DJ|{gVgpHk${0mu72(QJ#422g0F?~Emh6+drLKIJFusCMn zYt);;QA5XPTHV2nXwZ#Nx^a$aud&uJ0Es2Ux;6ysQl1gw44)zmPcH7i&CJtkcs5y* zkZ`u6$rZI&m6$cNV|aUUviAxP#t@&JPe6|KEX84#(v!(crG{^p+WXvj9vXg6{zyQ% z;-pSNmTl0YtC~jO<->Qs@-U!5H|hdLu3h6(;5a_zd19oI;!j9F`&6nKwIY<^_qN~2 zW5xA;6VHNCc*+1FYiud_4rOI6yW=y$DF3;1zXi{0qncE8!V#XgO_Px4Jx9KO+cfHb zPJOt+^TlX5wTf`e^;^>i)QhU)_B*9UlozDOC5&j$je3AlaIf*7(cjcRgp=o(mRTt; zqjC$G!N!U&8*qBOAY(mLEFqtlA1n)brI$a?*>4>1Dgi&wD`6am8YdLGNh3@#tIS0z z{BYyaRdeDAUKQhJv`bG>&smDkP0G%b@*ea;Kq z26^pW6Wx-^@{zq>Z=&C7rW-Dn6H z)%P0Dn`UQxB3yO5ENzXVP3af&f|)kIKRok;x6`yMGl0-}?q&!A^FgR9E@0Gj;)Cnl zMc%unFEeq3n@=BvWKlk>-RcWNnsIE(E!gmq%tW#@2;ICx=POW~J>5TIcFbU#-=2No zWk7>&Gy;seyT%dVJ7dn$YVuyV8mISart%*=&aTNLni6ePMPY>o4K%(=}-_ zPjT5eQ|PkbQawW=j=VRpAqcc^+yd zkjBNH>tLuoYa@Jeu+xWm2*Z4e&z&e=@}7uQYr?v|+IEye^ zrh6b7_0^kpE6@*n`L%dMj1djG(F8DB?=`|;l*2CxKkF+=w^=j$?vS2+%b(jxjPB?m+u)>#d;4S1w z`pW!+F7vlPu6_3BJ7)gl=x4%9_uMch%ukzFDtP!o@XuqEZN8F&nmK`lwX$T03^01_HMSqDJ*G)` z&v*5OC+0Wd1N$84VE^xL-#+toAAEAGhVaq7_eBzAyXuqwa@Rr1_VAA?zQ+gu9s5h5 zj1y!|qjsXGb!#vS#hvrN%=pGEXgB4AExw7cOPD`;TO03|Edu@||9-*8fCk+N1B^j? zjf)oX$0rFt+*d9>fY27@7*UWG<$tYq3i&oH&Yv(N{4RTcaTfJ=ZM*OXh&F#-&%(`rG{Dzi`xwnbD9wsl-vCK1kjt5Gu z_Oq~Y@G+u6Hy#9xiF=JUR=lUAi2@T2vge>IZG!bmnpOrZi<~|D9#%G|3W*~8KC)NQ ztQjH;8vRxYtT(xX`9rM^ouUxMJtD&KSTSH}ZmrQspU^R-o*{ z;6uinROcfv^0P)-L(AM5XdtIZY^MwQ>Nw-x|&NM5vDcqVr_4;^Y|mnHm~^3$;j~U*iZ{si53(8F8_n_D_Y*o zMcPX6N6HQgaM~&rxe#gp5~~te0T$X)I*_fafT4U6KLZ+cqcvbWvuh*)$B`r;Y8zD4 zOSJctSk3${7e*T8Ih1s9e21faIW;v%A} zfPQ2=Rw%>uZPTdjJE5B@VFCwje-=}T9-dZ_r>MfVwB<=iJ2~M~YRrsi(2X{Lv2xc) z1dK!h7dx#p!9?#0x0ds05n-Q0S9k1uMf}t|1p@3M&k%`zlfkQfSlU9Ipm%rmYAlwXcx0*Dn#cVt?VRIWe)~`kF75HjTc1Uelf= zC26Q9z<>tbXa^Wu{xi;1Y=_W>cPh3+1#a6G}U#ugBJLa$q9jR2iUp8#Egom)^@104{et9rbCQ$rI~`jvIF|u z8ZmqFMeAeq0mVaisW}b}2V!9(jA+n}_JHyFt}z%eA_U$!bd6J@i8+3P z9mZq~52gxycUUUBLp)ybTVV@In}i$ix65(AESII2AhRQ1xePIPid~TxE!#$$xe9kQ zkbQ3TKv2NZw!D~FASfKIN|nnfc*ni$m@L%q!|3WPCl54Yp z(pIyY2DadI>CClxD;1Q?yRWrnkUdPtAskCWHPL|Ei0;={NWr7T7(2Y)j5x;8;0?yT7 z!9r)5>Vw3VsS+hoC~YoLg9FUjPEFFwNU+-3yZQ>TL$E#uf>pasn}9Rwd|2(1_mW_X zb5V@|@tRkAj5Ad||GJtF!WpZM_stW$<@~Vb2(hcOyFCW2;dA3~(2nzK+T=(TMl|R~ zXTbPm*BAg80|on?ztpS~Z%sXZeFCegj~R|Ab&=4F4Hy?3c2TZ%CH8oYDK$cYNe!AV zu3Dx+>4I}E!L@zFJC%z{x6xWNeSsj(`BnFoIN-g%gBcC+ zp}i;5QD75&x~l$jjn)kb=Tk+EsB%9PcsJAKT4Csn ztP+ZIz1UyLyhxSO^8T0s4*U0$cP5r=mr?y?;3ppqrXs@n|bq5;(Ar(h7#5! znl|1Ia*HvkP1g`=c1vr1P28BixM4x14H6GeHoG;LCT2_vU30r~-i`Rd`-Vyo)Qn|Q zVgcqhW@eg|E!5|>eEu%+Q`P;r46NDA=Y;2@?)zc9hh!Mhpc~x*A|>jR9G8r3=!#W0$V*y5SnziyHG;>CNy*98rnSL+piBh?#bqNjwT5` zcRzAri}=&~)dm4e8?AEs((Hcu;OS$0LLc03UJM~_Pk&Z<4y6t8H2l(ZpS1`&-p7ar z-RJ=rfA2N^bYHtTNc<!n5#YqjWAEvQuu+Y!)?N2b+O-W6e4kKD`5B#s${w+>kAOxj~}smC?z zjsh1U1~lkKPr7lQY1fDVj$??hmdBmT$s~|4VzFVWm_ysf-^4hwOHHNLf2K`o_2Nz&R7Whdiu0nBZ&$J zT*X4|dTEa$o1Vw)zLcynqCq!$0YVdl&c#P^r=p>Dv55(UIosA z)e6FGUKQ=~BsJRQ=ni#mSHJI@<8{w@wPsKl>qTlWA!!NUewToCDbVIZfV@7s+^I_y ze&qGL{Tqq)Tly}K3jAbtdCS4iGzbYZpg}kK07l7OV<2#@1`CgP>vV*Y^k&A@E}&hN z^cxdk-oCD(jSqyMc}I5)k&J|w6YgSNzZtzxUG^?^({2tEe&bDhwMa7cp}ls4x{+w# zs7P;|JJSVbVcI*6D@`Qx>hIS+qTSmMo;e2c{^mY+v6B%Ey3rRf%I`6X?DJu|%0jZ7 zWq!|2rG0li$!qpe^XRzjF2dtubTx%!>norx52amjl4mr1LOhSO{1B1!NxC{ua;Q0w z2*Y}b(9Y6Pp9(Lxwk8o(pG()QNY1kwiGEbC*DlxV5VRFA@*OseXwZ#*fKh#~(ZFZ+ z+I5nfi1Bqa+Pl~7_xT;4Ki*%iY=~IL>SPZ8~p*J?tjL4 z75eWZ?K4;;(f4Yn2PxP$IdKt7+kHj-E`m0}=HSf}BH6wxo%czhHAk+0gZj^g9eK3r z#~L7rQ4z`a6X;SPMa&j<@}UEIBcG5;{Y(PhU`IqM{Or3*NiibTAJnLU1+iSCV87(R z`#tH5XwZ!TfYJ1SMzw9)Z^yjacA&@wzoT8>N%1u;8n)0N--Mv4e!t5>XL`j%I{j|l z2qPuU-bxC^1}h=2KZEhBku{yYfXyHO%!e~FvA zq%_~*&MGv5ly3Sa$KNW1={`Fn8gyeIV6@&f(#ASy#sHCNf47^>q|BPR4>zzO?OA6& zg8ef?<{sPf3z*K?Z0_zfpntgZ_*AmT;ycZzU2UcaJPYrjA+n}L4eVDkCC?IRt{rLIyJ}L zB?b+Do%etp8Q>Hlk9QCi3Gl^ql1__?Z$hyVq(W|9P{6T>?}QJc@&RYCBBWwJrA-%V zWX%~D;pPA=?ToDw)d_fvJx(f})6TBMh8~*D3TiH;x@;zvAvBW2F-!@V&9C7i!z`=HzELI&|V`t zklb^Yblxvh^F5Td>5&vnGw@^F{qbzkjw|AbApFcm7lwaehZ%K>mof8{rQoKO(i|%J`N84lluL|XE7!O_gzsE zu8vI`1Vz>0gY&x(P}GxGb)pPt(2Ze$F>TjK8?&MreZ}|?7I#mOi1R|)%4l>9sj~!* zh(TXiGZT|WWZZp68WNLDv81LM41K6rMx0MeTVE4XM_jw-MH=zfx)}^jXB{=G&p}M4 zTfZq3GeWG~dq8?xXZk4nW}W8RBsN9BJ0ySNl3TvtUlD3l@@`dH+MUSXfBT{hy?Ty2unF zHp}-}QulJm<*aibWyFvnm182`tji+>2uwqTgT8jB9x;q zcfv$+ynbF08w!(uWJ~%oKk~T_oAd4K$VO?H(=p}W?qaiHevfXDzKbn<#!`=wen@@N z3_Eu0_x4vtH0Z`Cz*xC!q>Vk&jNW3aVI_~nNWbdd=!~L|*ZdCN>JP&nfBL6Mj1o5R z_#|m({%h(R?1`8^o!f}8*C#Igvk?0ew)OZOd7t<`%zo%e!+ksZH^Ze*rtjMj=L%OF z@Fp_{@L;6SxuPs^j?!>X+7`qmjA+n}(SWgWk5ODKJanL+%vLW+F`(wYVK3zahZmfZ zV$l;<46ni|lexrIAs*0Eces{C`or&?`oKCb4h^Sm$wuZ0FhC_@^G@?>g5lvG^B%Jw z6}JfA!TloBwx~trQ)xHuIr5tkO8Hfs3XEvbjWK|+Wv>w)0mVm>g%;c~7tsaE!u@JF z5rG9!T!Z3)5wZADvY0r6au0i&O&kOPMVvit1WFZ;jc6nokR|IAAv4g^bF@`=T@g6i zt)7s0dc-*4B3XJN8}bQV*m0oQ92W7d@CDxkaRxN###q33eXlV$l7+}YmJ`oMb5M)2 zpokre|funN#(dnK4NGBTn#Pl#Q-DKeS3L{_Q4fHg;-QC2PW z1kvtV{e?TluST|z?8xd1om7A7neQ5-{>_oYB^sjM;tXiejd6gnd)Md#T#ugOy^-^z zn`G^Pd(bo}ZMQ&hC@k_%$+p-JaY7V`EKY{jlhLKv(pp_)^k@{cbX?-1_(YT?IhRaZ z1%mFNo{cr=PeMi|oNbk|5nqYQB)=yci@(I;(B}+|chEag=g(z>KQN*}H^u|T`+JS= zqplD7kj(-WO=BUXshXZBeKcb!wJ|rJp*oJ{ zXh?-@Ey1D7k1oGsy;(3CZBqVOZb?Eg+Ht6yY!@h!rcSLmZLe91h)%9}tZ+h{0S&q_ z0WjkC7$sz)j}84MJ2uF7+d(V+oN_96qT4I0lvE@%qp`zLWS3_;U&63ehO`$vrP0r- z;*>`uOrqZmkCELaU|$YXtJyq2m%uSRw8xTk3HunSCx&FN24~%BboHD!qN6{?s)k!# zj1djGF%d96*=zKP@py8H?EB0=?IyOS!>{WGJSMZ|g~nrvh?x8lPBQI5t9uw)n-#dz zvl(-vwnr;UA|>YDNIDr2i0)oP*V%`_?&rk3sw>ptmN**oX=Irk)^PmGA8fsHcqtAX zD^l+dxkVei{r}#e8{CbOSj~^C>*`Si?WPZHd;{(lK#zLPJM7oZ48Mh@4rbJuFTev`S!N zabu^*$Y(u0anPpN0c*mq5>Mh=W<(2YpIxVYE&F!tNn2Qo@x;A<|n*&YpA1;t4> zXB&T(coL`n)R&wdIQF%PdR{aG@wSULCCA0IN@6}P?CB$N*0UwOJLvOovYS7{fagvl ziPv$}*)jw%;gh2N_4bNU+G0_!swQe4Bi-k?A%o`jTJnF{Yh(rAu{I8j_%R-HB*&-Xhv2hm__&Fu>6TzRl|il!^4x00w)@q%s+T52v=_SYF`_{?rUJ$< zdyT;fDN`%ty2jA0htv+``twlDgqn_Ec#mX!LhG~xxk(bKzku%e-n3%WpD;|@?>Ja8 zD`8<8L%tAnIO7ZU3i+beK}5ozEC1a8N#-SjW+cd$8&CK0K(E%e9JAd_G`KqLc}0=| z4Z0Bp7=Q0ImM79S5F@uOR`sf&uVuB5xs)a*T)X1!B6*&+9B`9-Rr1m|E9!OoYdl`y z#Pip)eLhQGPwbraBX>65G(bQ*MY}=*`V%KucbePj>(sQAOOj;7VqXWfukRP5!A`fHNq6VepkOj}U$-`2L zcbw4Or=RdtgOE46`(lzL8PK2`(*dL49;1{t^8T|-@@O#hdla?DZ_Kr^8TqQGBu+@m z4Eg!lbMp9-h2e4NZRLscmtn~ww}a#Rq#TnK79D6i2EeoGvA4fZf4nkEn_Z@p7$)VH zOnZ+*o(m4l>ZbOR=gn^-lXLGdBbgb|pc^v)qvWoUK4vwz9Viu*e0E8a{H!VV&KUYm z?c%Z9JITFw=94?6P|5hEeDd;A`u7cNU)IXlgXZLSeOFT5rH&|&D@acP8nXQB7Y3- z{gH%yQ2WVsxip3PU@*Hx>T$~7l|N)k6aGg5wLg}6b`6}W)ZcViPl^Ezx-knds_!+9 zrs_S9C2vVh7+pd?G~8Nw*Pj~rFzLv=)M9G<^9k~g;FawA*hg$XwJ7k^vyUutPDpK} zHoq_-|89DpJwtsow|(sUCT)~e=(vK^Rw`|JF8R;WkG{|7#~ptk{>e%G^?2jNh!g`F zbYnJP)ZI1G<{;6|RX^~46x&PC;C?X6PfqMWArmhP7^OAPmzxUaMVYC$h}d@u zO?HsrmxrOxlMiwH05hOLHy#Fz)_aXf=q{#!L8;~x%VzAL7&u&Y89g}&=eh_!gnr3{ z8$7Ul^kg4&a6<;8gG}Qbk_Fj-3(`cGH3sFuMJFfGLw)iTgPk<-=ITKu zM9shl>TsR1C9F9OJxt^)12dpOHy#0u&U=j)(@rq|9aLRz-OmPn5~o&fqnXzJq+Y-X z+?CeD5;v#`##qT>M~pOII*q118$k*#f$yiiWtkk*K|HiFrH-;gWIYh+0;86~dEmix zX;#xg-R38`zR=NmJ(S;0y4@JRs0x??4Z86tVD#E+oJsd$Z5uRLp2wwP$NCHhLQ2yQ zkFAT1gI}eeX5$$&0k7kVsZR?{`J*+{Z#=y#kq-Wp{(voO5Z3%AW(Y vhdOn|`D6i@_MT)$?v^wpgTQfm&{bNWNfkPs?K)A?pYd+`s)9cw8vFha9P&T< literal 0 HcmV?d00001 diff --git a/imap_processing/mag/l0/decom_mag.py b/imap_processing/mag/l0/decom_mag.py index 2be053df9..b895801b6 100644 --- a/imap_processing/mag/l0/decom_mag.py +++ b/imap_processing/mag/l0/decom_mag.py @@ -71,58 +71,117 @@ def export_to_xarray(l0_data: list[MagL0]): ---------- l0_data: list[MagL0] A list of MagL0 datapoints + + Returns + ------- + norm_data : xr.Dataset + xarray dataset for generating burst data CDFs + burst_data : xr.Dataset + xarray dataset for generating burst data CDFs """ # TODO split by mago and magi using primary sensor - # TODO split by norm and burst - norm_data = defaultdict(list) - burst_data = norm_data.copy() - - for datapoint in l0_data: - if datapoint.ccsds_header.PKT_APID == Mode.NORMAL: - for key, value in dataclasses.asdict(datapoint).items(): - if key != "ccsds_header": - norm_data[key].append(value) - if datapoint.ccsds_header.PKT_APID == Mode.BURST: - burst_data["SHCOARSE"].append(datapoint.SHCOARSE) - burst_data["raw_vectors"].append(datapoint.VECTORS) + norm_data = [] + burst_data = [] + + for packet in l0_data: + if packet.ccsds_header.PKT_APID == Mode.NORMAL: + norm_data.append(packet) + if packet.ccsds_header.PKT_APID == Mode.BURST: + burst_data.append(packet) + + norm_dataset = None + burst_dataset = None + + if len(norm_data) > 0: + norm_dataset = generate_dataset(norm_data) + if len(burst_data) > 0: + burst_dataset = generate_dataset(burst_data) + + return norm_dataset, burst_dataset + + +def generate_dataset(l0_data: list[MagL0]): + """ + Generate a CDF dataset from the sorted L0 MAG data. + + Used to create 2 similar datasets, for norm and burst data. + + Parameters + ---------- + l0_data : list[MagL0] + List of sorted L0 MAG data. + + Returns + ------- + dataset : xr.Dataset + xarray dataset with proper CDF attributes and shape. + """ + vector_data = np.zeros((len(l0_data), len(l0_data[0].VECTORS))) + shcoarse_data = np.zeros(len(l0_data)) + + support_data = defaultdict(list) + + for index, datapoint in enumerate(l0_data): + vector_len = len(datapoint.VECTORS) + if vector_len > vector_data.shape[1]: + # If the new vector is longer than the existing shape, first reshape + # vector_data and pad the existing vectors with zeros. + vector_data = np.pad( + vector_data, + ( + ( + 0, + 0, + ), + (0, vector_len - vector_data.shape[1]), + ), + "constant", + constant_values=(0,), + ) + vector_data[index, :vector_len] = datapoint.VECTORS + + shcoarse_data[index] = calc_start_time(datapoint.SHCOARSE) + + # Add remaining pieces to arrays + for key, value in dataclasses.asdict(datapoint).items(): + if key not in ("ccsds_header", "VECTORS", "SHCOARSE"): + support_data[key].append(value) # Used in L1A vectors - direction_norm = xr.DataArray( - np.arange(len(norm_data["VECTORS"][0])), + direction = xr.DataArray( + np.arange(vector_data.shape[1]), name="Direction", dims=["Direction"], attrs=mag_cdf_attrs.direction_attrs.output(), ) - norm_epoch_time = xr.DataArray( - [calc_start_time(shcoarse) for shcoarse in norm_data["SHCOARSE"]], + epoch_time = xr.DataArray( + shcoarse_data, name="Epoch", dims=["Epoch"], attrs=ConstantCoordinates.EPOCH, ) # TODO: raw vectors units - norm_raw_vectors = xr.DataArray( - norm_data["VECTORS"], + raw_vectors = xr.DataArray( + vector_data, name="Raw_Vectors", dims=["Epoch", "Direction"], attrs=mag_cdf_attrs.mag_vector_attrs.output(), ) # TODO add norm to attrs somehow - norm_dataset = xr.Dataset( - coords={"Epoch": norm_epoch_time, "Direction": direction_norm}, + output = xr.Dataset( + coords={"Epoch": epoch_time, "Direction": direction}, attrs=mag_cdf_attrs.mag_l1a_attrs.output(), ) - norm_dataset["RAW_VECTORS"] = norm_raw_vectors - - # TODO: retrieve the doc for the CDF description (etattr(MagL0, "__doc__", {})) + output["RAW_VECTORS"] = raw_vectors - for key, value in norm_data.items(): + for key, value in support_data.items(): # Time varying values if key not in ["SHCOARSE", "VECTORS"]: - norm_datarray = xr.DataArray( + output[key] = xr.DataArray( value, name=key, dims=["Epoch"], @@ -135,6 +194,5 @@ def export_to_xarray(l0_data: list[MagL0]): display_type="no_plot", ).output(), ) - norm_dataset[key] = norm_datarray - return norm_dataset + return output diff --git a/imap_processing/mag/l1a/mag_l1a.py b/imap_processing/mag/l1a/mag_l1a.py index bbae37098..548e9ef5f 100644 --- a/imap_processing/mag/l1a/mag_l1a.py +++ b/imap_processing/mag/l1a/mag_l1a.py @@ -1,23 +1,34 @@ """Methods for decomming packets, processing to level 1A, and writing CDFs for MAG.""" +import logging from pathlib import Path from imap_processing.cdf.utils import write_cdf from imap_processing.mag.l0 import decom_mag +logger = logging.getLogger(__name__) -def mag_l1a(packet_filepath, output_filepath): + +def mag_l1a(packet_filepath, output_filepath_norm, ouptput_filepath_burst): """ Process MAG L0 data into L1A CDF files at cdf_filepath. Parameters ---------- - packet_filepath: + packet_filepath : Packet files for processing - output_filepath: - Directory for output + output_filepath_norm : + Full directory and filename for raw-norm CDF file + ouptput_filepath_burst : + Full directory and filename for raw-burst CDF file """ mag_l0 = decom_mag.decom_packets(packet_filepath) - mag_datasets = decom_mag.export_to_xarray(mag_l0) + mag_norm, mag_burst = decom_mag.export_to_xarray(mag_l0) + + if mag_norm is not None: + write_cdf(mag_norm, Path(output_filepath_norm)) + logging.info(f"Created CDF file at {output_filepath_norm}") - write_cdf(mag_datasets, Path(output_filepath)) + if mag_burst is not None: + write_cdf(mag_burst, Path(ouptput_filepath_burst)) + logging.info(f"Created CDF file at {output_filepath_norm}") diff --git a/imap_processing/mag/mag_cdf_attrs.py b/imap_processing/mag/mag_cdf_attrs.py index 52f1350d3..9a9e08a38 100644 --- a/imap_processing/mag/mag_cdf_attrs.py +++ b/imap_processing/mag/mag_cdf_attrs.py @@ -26,7 +26,9 @@ ) mag_l1a_attrs = GlobalDataLevelAttrs( + # TODO: data type should include "norm" and "burst" L1A-norm>Level-1A-normal-rate "L1A>Level-1A", + # Should also include data type logical_source="imap_mag_l1a", logical_source_desc="IMAP Mission MAG Instrument Level-1A Data.", instrument_base=mag_base, @@ -50,6 +52,8 @@ # TODO: display type, catdesc, units, format, label_axis +# TODO: update descriptor to be more accurate for L1A raw +# TODO: does raw value need "counts" mag_vector_attrs = ScienceAttrs( validmin=GlobalConstants.INT_FILLVAL, validmax=GlobalConstants.INT_MAXVAL, diff --git a/imap_processing/tests/mag/test_mag_decom.py b/imap_processing/tests/mag/test_mag_decom.py index 5f31be68c..69f73bade 100644 --- a/imap_processing/tests/mag/test_mag_decom.py +++ b/imap_processing/tests/mag/test_mag_decom.py @@ -1,8 +1,10 @@ from pathlib import Path import pandas as pd +from cdflib.xarray import cdf_to_xarray from imap_processing.cdf import global_attrs +from imap_processing.cdf.utils import write_cdf from imap_processing.mag.l0.decom_mag import decom_packets, export_to_xarray @@ -12,7 +14,6 @@ def test_mag_decom(): l0 = decom_packets(burst_test_file) expected_output = pd.read_csv(current_directory / "mag_l0_test_output.csv") - for index, test in enumerate(l0): assert test.ccsds_header.PKT_APID == expected_output["PHAPID"][index] assert test.ccsds_header.SRC_SEQ_CTR == expected_output["PHSEQCNT"][index] @@ -35,15 +36,50 @@ def test_mag_decom(): assert len(l0) == len(expected_output.index) -def test_mag_raw_cdf(): +def test_mag_raw_xarray(): current_directory = Path(__file__).parent burst_test_file = current_directory / "mag_l0_test_data.pkts" l0 = decom_packets(str(burst_test_file)) - output_data = export_to_xarray(l0) + norm_data, burst_data = export_to_xarray(l0) required_attrs = list( global_attrs.GlobalInstrumentAttrs("", "", "").output().keys() ) - assert all([item in list(output_data.attrs.keys()) for item in required_attrs]) - assert all([item is not None for _, item in output_data.attrs.items()]) + assert all([item in list(norm_data.attrs.keys()) for item in required_attrs]) + assert all([item is not None for _, item in norm_data.attrs.items()]) + + assert all([item in list(burst_data.attrs.keys()) for item in required_attrs]) + assert all([item is not None for _, item in burst_data.attrs.items()]) + + expected_norm_len = 17 + assert norm_data.dims["Epoch"] == expected_norm_len + + expected_burst_len = 19 + assert burst_data.dims["Epoch"] == expected_burst_len + + +def test_mag_raw_cdf_generation(tmp_path): + current_directory = Path(__file__).parent + test_file = current_directory / "mag_l0_test_data.pkts" + l0 = decom_packets(str(test_file)) + + norm_data, burst_data = export_to_xarray(l0) + + test_data_path_norm = tmp_path / "mag_l1a_raw-normal_20210101_20210102_v01-01.cdf" + + assert not test_data_path_norm.exists() + output = write_cdf(norm_data, test_data_path_norm) + assert test_data_path_norm.exists() + + input_xarray = cdf_to_xarray(output) + assert input_xarray.attrs.keys() == norm_data.attrs.keys() + + test_data_path_burst = tmp_path / "mag_l1a_raw-burst_20210101_20210102_v01-01.cdf" + + assert not test_data_path_burst.exists() + output = write_cdf(burst_data, test_data_path_burst) + assert test_data_path_burst.exists() + + input_xarray = cdf_to_xarray(output) + assert input_xarray.attrs.keys() == burst_data.attrs.keys()