diff --git a/am_flow/.gitignore b/am_flow/.gitignore new file mode 100644 index 0000000..03e93f2 --- /dev/null +++ b/am_flow/.gitignore @@ -0,0 +1,9 @@ +.httpdocroot +.upstreamhttpserverpid +.httpserverpid +.ftpserverpid +.flowpostpid +flow_save/ +__pycache__/ +on_*.py +^sender/ diff --git a/am_flow/config/flow/amserver-flow_f01.conf b/am_flow/config/flow/amserver-flow_f01.conf new file mode 100644 index 0000000..bb26bce --- /dev/null +++ b/am_flow/config/flow/amserver-flow_f01.conf @@ -0,0 +1,35 @@ +#------------------ PREREQUISITES +callback_prepend gather.am +# Add AHL headers to filename +# callback rename.raw2bulletin +callback post.message + +#------------------ DESTINATION +directory ${TESTDOCROOT}/bulletins_to_send/${YYYYMMDD}/${T1}${T2}/${CCCC}/${GG} +# We want to locally download the files +download on +# directory /datalocal/am/bulletins/ +destination am://${FLOWBROKER}:5005 + +post_topicPrefix v03.post +post_broker ${MQP}://tfeed@${FLOWBROKER} +post_exchange xpublic +post_baseUrl file://sarra@${FLOWBROKER}/ +post_base_dir ${TESTDOCROOT} + +# ----------------- FILTERING +accept .* + +# ----------------- OPTIONS +mirror False +sum sha512 +# Some bulletins have iso-8859-1 encoded characters that UTF-8 can't resolve. +inputCharset iso-8859-1 + +#------------------ LOGGING +debug on +# logLevel debug +# messageDebugDump on + +#------------------ INCLUDES +include stations.inc diff --git a/am_flow/config/flow/stations.inc b/am_flow/config/flow/stations.inc new file mode 100644 index 0000000..192bf27 --- /dev/null +++ b/am_flow/config/flow/stations.inc @@ -0,0 +1,221 @@ +mapStations2AHL FCCN61 CWAO CQMD CQPB +mapStations2AHL FTCN31 CWAO CBBC CYAZ CYBL CYCD CYCG CYKA CYLW CYPR CYQQ CYQZ CYVR CYWH CYWL CYXC CYXS CYXT CYXX CYYD CYYF CYYJ CYZP CYZT CYZY CZMT +mapStations2AHL FTCN32 CWAO CFP7 CYBW CYDL CYDQ CYED CYEG CYKJ CYLL CYMM CYOD CYOJ CYPE CYPY CYQF CYQL CYQU CYSF CYVT CYXH CYXJ CYYC CYYE CYZH CYZU CZVL +mapStations2AHL FTCN33 CWAO CYAB CYBB CYBK CYCB CYCO CYCY CYDA CYDB CYEK CYEU CYEV CYFS CYGH CYGT CYHI CYHK CYHY CYIO CYLK CYLT CYMA CYPC CYOA CYOC CYQH CYRA CYRB CYRT CYSM CYSY CYTE CYUB CYUT CYVM CYVQ CYUX CYWE CYWJ CYXP CYXY CYYH CYZF CYZS CYZW CZFA CZFM +mapStations2AHL FTCN34 CWAO CYAT CYBR CYDN CYEN CYER CYGQ CYGX CYHD CYIV CYMJ CYNE CYPA CYPG CYPL CYQD CYQK CYQR CYQT CYQV CYQW CYRL CYTH CYTL CYVC CYWG CYXE CYXL CYYL CYYN CYYQ CZMD CZSJ +mapStations2AHL FTCN35 CWAO CYAM CYBN CYGK CYHM CYKF CYKZ CYLD CYMO CYQA CYPQ CYQG CYSB CYSN CYSP CYTR CYTS CYTZ CYVV CYWA CYXR CYXU CYXZ CYOO CYYB CYYU CYYZ CYZE CYZR +mapStations2AHL FTCN36 CWAO CYAH CYAS CYBC CYBG CYBX CYFB CYGL CYGP CYGV CYGW CYHA CYHU CYIK CYKG CYKL CYKQ CYML CYMT CYMX CYNA CYND CYOW CYPH CYPX CYOY CYQB CYRJ CYRQ CYSC CYTQ CYUL CYUY CYVO CYVP CYWK CYYY CYZG CYZV +mapStations2AHL FTCN37 CWAO CWSA CYAW CYCX CYFC CYGR CYHZ CYQI CYQM CYQY CYSJ CYYG CYZX CZBF +mapStations2AHL FTCN38 CWAO CWWU CYAY CYCA CYDF CYDP CYJT CYMH CYQX CYYR CYYT CZUM +mapStations2AHL FTCN61 CWAO CQMD CQPB +mapStations2AHL FTCN99 CWAO CZZA CZZB +mapStations2AHL SACN00 CWAO01 MBR QAE QAL QAP QAS QAU QBA QBC QBD QBE QBF QBK QBL QBM QBO QBQ QBR QBW QCA QCB QCD QCE QCH QCM QCN QCO QCP QCQ QCR QCS QCV QCY +mapStations2AHL SACN00 CWAO02 QDA QDB QDC QDF QDI QDJ QDK QDL QDM QDN QDO QDP QDU QDZ QEA QEC QED QEM QES QFA QFL QFO QFR QFT QGA QGD QGI QGO QGS QHH QHI QIO QJE QJO QJR QKM +mapStations2AHL SACN00 CWAO03 MPD QLA QLB QLC QLE QLF QLG QLL QLM QLP QLS QLV QMA QMC QMD QME QMG QMI QMM QMO QMR QMS QMT QMU QMW QMX QNA QND QNI QNO +mapStations2AHL SACN00 CWAO04 MRG QOB QOI QOM QON QOS QOU QPA QPC QPE QPK QPM QPO QPR QPS QPT QPU QRB QRC QRK QRO QRP QRT QSA QSE QSK QSS QST QSU QTI QTM QTV +mapStations2AHL SACN00 CWAO05 MBB MBM MCM QMB QMQ QUC QUE QVA QVG QVI QWA QWE QWG QWJ QWP QXA QXE QXI QXJ QXK QXM QXN QXP QXS QYE QYN QYO QYR QYS QYY QZE SEN TAF TAL TCW +mapStations2AHL SACN00 CWAO06 AAH AKE AMK AMN ASI MAA MAE MAF MAG MAI MAK MAL MAP MAR MAS MAT MAU MBA MBC MBD MBE MBF MBI MBK MBL MBM MBO MBP MBQ MBS MBT MBU MBW MBY MCA MCB MCD MCE MCH MCL MCN MCO MCP MCQ MCR MCS MCU MCV MCX MCY MDA MDB MDC MDF MDI MDJ MDK MDL MDM MDN +mapStations2AHL SACN00 CWAO07 MDO MDP MDU MDV MDW MDZ MEA MEC MED MEM MEN MEO MES MFA MFE MFL MFN MFO MFQ MFR MFT MFU MFW MFY MFZ MGA MGD MGF MGI MGK MGO MGP MGS MGT MGV MGX MHA MHC MHH MHI MHJ MIK MIM MIO MIQ MIR MJE MJO MJR MJU MKB MKG MKI MKM MKW MLB MLC MLD MLE +mapStations2AHL SACN00 CWAO08 MLF MLG MLH MLL MLM MLO MLP MLS MLV MLZ MMA MMB MMC MMD MME MMG MMI MMK MMM MMO MMP MMQ MMR MMS MMT MMU MMW MMX MMZ MNA MND MNE MNI MNO MNY MOA MOB MOI MOM MON MOP MOS MOU MPA MPC MPE MPI MPK MPM +mapStations2AHL SACN00 CWAO09 MPN MPO MPP MPR MPS MPU MPW MPX MRA MRC MRE MRK MRL MRM MRN MRO MRP MRS MRT MRW MRX MRZ MSA MSE MSK MSL MSM MSN MSS MST MSU MSY MTE MTI MTM MTO MTS MTV MUE MUT MVA MVG MVI MVN MVO MWA MWE MWG MWH MWJ MWP MWT MXA MXE MXI MXJ +mapStations2AHL SACN00 CWAO10 MXM MXN MXP MXS MXT MYA MYE MYN MYO MYR MYS MYY MZE +mapStations2AHL SACN00 CWAO CWDH CWES CWVA CWZB CXWT HDR QBT QCU QCX QHA QIQ QJU QLO QWH QYC WBS WED XSN YDL YPS YSE ZEM ZSJ ZTS +mapStations2AHL SACN00 CWAOSA AAY ABC ABN ABX ACY AED AEK AFL AGV AHU AIO ALW ANA APH APQ AQL AQZ ASC ASR ATE AUX AVM AVO AWA AXD AXP AXH AZP BDB GSC OZE PCP THK UPR VDQ VLW VQA WQB WTQ ZCL +mapStations2AHL SACN00 PANAM A0T A1T A2T A3T A4T A5T A6T A7T A8T A9T L1B L1C L1D L1E L1F L2A L2B L2C L2D L2F L3A L3B L3C L3D L3E L3F L3G L3H L3I L4A L5A L5B L6A L6B W5Z Z1D Z1E Z1N Z1W Z2D Z2E Z2W Z3E Z3N Z3W Z4E Z4N Z4W Z5E Z5N +mapStations2AHL SACN11 CWAO ZCX ZMH ZMT ZOY +mapStations2AHL SACN21 CTST ADB AND CADB CAND CDNA CDOW CDOX CGEO CNEK CQOY CWSE CXYZ CYHN CZTS CZVM CZXT CZXX DNA DOW DOX GEO NBU NEK PDK QOY TIZ TM1 TTA TVQ UQA UQB UQC WSE XYZ ZHM ZUL ZVM ZXT ZXX ZZD ZZE ZZF ZZG ZZH ZZI ZZK ZZL ZZM ZZZ +mapStations2AHL SACN22 CTST CBBD CTSN CXZR CZAH CZDQ CZHQ CZQV CZQW CZYL DOY DOZ TSN XZR YER ZAH ZDQ ZQW +mapStations2AHL SACN31 CWAO COLL CYAW CYBG CYEG CYFB CYFC CYGL CYHZ CYJT CYMX CYOW CYQB CYQG CYQM CYQQ CYQX CYQY CYTR CYUL CYVR CYWG CYXX CYYC CYYJ CYYQ CYYR CYYT CYYZ CYZF CYZV CYZX +mapStations2AHL SACN40 CWAO COLL CYWH +mapStations2AHL SACN41 CWAO COLL CBBC CYBD CYCD CYCG CYDA CYDL CYGE CYIN CYKA CYLW CYMA CYOC CYPW CYQH CYQU CYSD CYWL CYXJ CYXT CYXY CYYD CYYE CYYF CYZW CYZY CZFA CZMT +mapStations2AHL SACN42 CWAO COLL CYAZ CYBL CYBV CYCP CYCQ CYDB CYDC CYDQ CYEV CYFO CYHE CYIV CYKY CYMD CYPR CYPS CYQD CYQZ CYRV CYSE CYUA CYVL CYXC CYXE CYXQ CYXS CYZP CYZT +mapStations2AHL SACN42 CZZA CZZA CZZB CZZC ZZA ZZB ZZC +mapStations2AHL SACN43 CWAO COLL CWHN CWLO CYAB CYBK CYBQ CYBR CYBU CYDN CYEN CYET CYGX CYKJ CYLL CYMM CYNE CYOJ CYPA CYPG CYPY CYQF CYQL CYQR CYQW CYVC CYYL CYYN CYZH CZVL +mapStations2AHL SACN44 CWAO COLL CWIQ CWRF CWVD CYBW CYCY CYED CYEK CYIO CYLJ CYLT CYMJ CYOD CYPE CYQV CYSF CYSM CYTH CYUB CYUJ CYUS CYUX CYVT CYXH CYZU +mapStations2AHL SACN45 CWAO COLL CYBB CYCB CYCO CYCS CYFS CYGH CYGT CYHI CYHK CYHY CYJF CYKD CYLC CYLK CYOA CYPC CYRA CYRT CYSY CYTE CYUT CYVM CYWE CYWJ CYWY CYXN CYXP CYZS +mapStations2AHL SACN46 CWAO COLL CYAT CYBC CYBN CYCK CYER CYFR CYGK CYGQ CYHM CYKF CYKL CYKP CYKZ CYLD CYLH CYOO CYPL CYPO CYQA CYQK CYRB CYRQ CYSB CYSC CYSN CYVO CYSP CYVQ CYXL CYXU CYXZ CYYB CYYH CYYU CYYW CZMD CZSJ +mapStations2AHL SACN47 CWAO COLL CGGA CYAM CYFT CYGP CYGR CYHD CYHU CYKO CYMO CYNA CYND CYPQ CYQT CYRJ CYRL CYTL CYTS CYTZ CYVP CYVV CYWA CYXR CYYY CYZE CYZR +mapStations2AHL SACN48 CWAO COLL CYAH CYAS CYBX CYGV CYHA CYIK CYKG CYKQ CYLA CYMH CYML CYMT CYMU CYNM CYOY CYPH CYPX CYQI CYSK CYUY CYWK CYZG +mapStations2AHL SACN49 CWAO COLL CYAY CYCX CYDF CYDP CYGW CYLU CYNC CYSJ CYTQ CYYG +mapStations2AHL SACN50 CWAO COLL NCN VAS VBB VBD VBR VCL VCW VFN VFS VGE VLY VME VMO VNB VOC VOD VOI VOM VOO VOP VOQ VOU VPE VPX VQZ VRA VSL VTF VTS VVR VXC VXS VYD VZP VZY WFG WGP WHC WPN WPU ZKA +mapStations2AHL SACN51 CWAO COLL CWAE CWCA CWIL CWKM CWKW CWLI CWLX CWLY CWOB CWPX CWRX CWSW CWUP CWUW CWVH CWYK CWZZ CYCA CZBF CZCP CZST +mapStations2AHL SACN52 CWAO COLL CWEU CWFD CWGZ CWKP CWRH CWSA CWTU CWWU CWXR CWYM CYPD CZEM CZFM CZFN CZPC CZUM +mapStations2AHL SACN54 CWAO COLL VXY WUM XQH +mapStations2AHL SACN58 CWAO COLL ERU EWS STH WDH WLT WZB XWT ZHD ZQA ZXR +mapStations2AHL SACN59 CWAO COLL NLB NLD NNR NPK NTK PAA PAH PBA PBE PBH PBL PBR PBS PBT PBV PCA PCM PCR PEA PEH PFI PFO PGE PGL PIR PMR PQL PQU PRA PRE PRL PRO PRR PRY PSD PSP PST PSV PTH PTR PTU PVA PWB PXL WCT WLB XAK XDB XFM XHD XHP XMG XMO XOL XPL XSC XSL XSP XTH XVM XVW ZMU ZOL ZPS +mapStations2AHL SACN60 CWAO COLL CFP7 NES NTL PAC PAF PAT PAV PAZ PBB PBC PBD PBM PBN PBU PCC PCE PCF PCH PCI PCK PCL PCN PCS PCY PDA PDB PDO PDU PEL PET PFA PFB PFE PFF PFM PFR PGB PGC PGI PGT PHA PHE PHR PHU PHW PIA PII PJT PKA PKG PKI PKK PKO PKR PKY PLA PLC PLF PLG PLY PLZ PMA PMB PML PMN PNB PNS PNU PNX PPL PPN PPT PPV PRD PRF PRH PRS PRW PSA PSH PSL PSQ PSR PSW PTA PTC PTE PVI PVR PWT PXB PXC PXH PXS PXX PXZ PYT PZC PZH PZK PZL PZN PZW +mapStations2AHL SACN61 CWAO XTR YAW YBG YEG YFB YFC YGL YHZ YJT YMX YOW YQB YQG YQM YQQ YQX YQY YTR YUL YVR YWG YXX YYC YYJ YYQ YYR YYT YYZ YZF YZV YZX +mapStations2AHL SACN63 CWAO COLL NPV NVQ WMT XAR ZEV ZFS ZHY ZSM +mapStations2AHL SACN66 CWAO COLL PIF PIL PJM PLS PMK POX PPR PQW PRJ PUN PWX PYS WBD WBU WDC WEH WIK WKO WOY XBK ZHB +mapStations2AHL SACN70 CWAO01 WAD WAN WAU WCV WCZ WEM WES WFM WIF WJG WLA WMR WMS WNJ WPI WTI WVB WZM YWH +mapStations2AHL SACN70 CWAO COLL PBO PGF PGH PJP PLE POY PQD PSU PTG PYQ WEX WJD WOO XMN XWG WSZ +mapStations2AHL SACN71 CWAO BBC YBD YCD YCG YDA YDQ YGE YKA YLW YMA YMM YOC YPW YQH YQU YSD YWL YXJ YXT YXY YYD YYE YYF YZW YZY ZFA +mapStations2AHL SACN72 CWAO YAZ YBL YBV YCP YCQ YDB YDC YEV YFO YHE YIV YKY YMD YPR YQD YQZ YRV YUA YXC YXE YXQ YXS YZP YZT +mapStations2AHL SACN73 CWAO WHN WLO YAB YBK YBQ YBR YBU YDN YEN YET YGX YKJ YLL YNE YOJ YPA YPG YPY YQF YQL YQR YQW YVC YYL YYN YZH +mapStations2AHL SACN74 CWAO01 WIQ WRF WVD YBW YCY YED YEK YIO YLJ YLT YMJ YOD YPE YQV YSF YSM YTH YUB YUJ YUS YUX YVT YXH YZU +mapStations2AHL SACN74 CWAO COLL NBB NBI NCD NCO NDT NEK NFR NGC NGH NPL NSG NZS WIC XBL XCM XFB XUX ZCY ZHK ZLT ZVM +mapStations2AHL SACN75 CWAO YBB YCB YCO YCS YFS YGH YGT YHI YHK YHY YJF YKD YLC YLK YOA YPC YRA YRT YSY YTE YUT YVM YWJ YWY YXN YXP YZS +mapStations2AHL SACN76 CWAO YBC YBN YFR YGK YGQ YHM YKF YKL YKZ YLD YPL YQA YQK YRB YSB YSC YSN YSP YVO YVQ YXL YXU YXZ YYB YYH YYU ZMD +mapStations2AHL SACN77 CWAO YAM YFT YGP YGR YHD YHU YKO YMO YNA YND YPQ YQT YRJ YRL YTL YTS YTZ YVP YVV YWA YXR YYY YZE YZR +mapStations2AHL SACN78 CWAO01 YAH YAS YBX YGV YHA YIK YKG YKQ YLA YMH YML YMT YMU YNM YOY YPH YPX YQI YSK YUY YWK YZG +mapStations2AHL SACN78 CWAO COLL OGE OSL OTL OTR TAE TAG TAL TBF TBO TBT TCK TCW TGT TKG TKR TLS TMS TNK TPM TPQ TRA TRY TSB TTR TUX TWL TWN TXR TZE TZN TZR WAJ WBE WKK WLS WNB WNL WPC WPS WWX XBI XRG XZC YLH ZEL +mapStations2AHL SACN79 CWAO YAY YCX YDF YDP YGW YLU YNC YSJ YTQ YYG +mapStations2AHL SACN81 CWAO WAE WCA WKM WKW WLI WLX WLY WOB WPX WRX WSW WUP WUW WVH WZZ YCA ZBF ZCP ZST +mapStations2AHL SACN82 CWAO1 WEU WFD WGZ WKP WRH WSA WTU WWU WXH WXR WYM ZFM ZFN ZPC ZUM +mapStations2AHL SACN82 CWAO COLL EUQ GAH GBC GCH GKL GKM GMG GRQ GVO MAN MBR MCT MCW MFJ MFM MFX MGB MGZ MHB MHN MHW MIN MJI MJK MLA MLI MLU MLT MMY MNT MPD MPL MQK MQL MQP MQQ MQU MQV MRF MRI MRU MRY MSB MSC MSI MSR MSX MTH MUS MWD MYT NSQ WDM WHP +mapStations2AHL SACN85 CWAO COLL WBK WHO WPK WWE YGS +mapStations2AHL SACN86 CWAO COLL WBE WNL WPO WPS WSI WWX WWZ +mapStations2AHL SACN87 CWAO COLL WEK WEQ WJD WLP WPU WRO WWL WWS WZG +mapStations2AHL SACN88 CWAO COLL AUU AAW ABB ABF ABH ABR ABT ACM ACP ACQ ADL ADN ADS AEI AFC AFY AGL AHD AHK AHR AHT AHW AJT AJW AKC AMR AMS ANH AOH AOS APB APR AQY ARP ASB ASF AVA AWR EDA ESB EQI ERM EWD OSM WAR WBK WBV WDA WDO WHO WNS WRN WSS WXI WZF WZN WZU XGD XKT XNP ZCR ZDB ZSP +mapStations2AHL SACN89 CWAO COLL WAF WAG WAQ WAS WAV WAY WBA WBG WBJ WBM WBO WBS WBT WBY WBZ WCF WCG WCH WCI WCJ WCO WCQ WDB WDD WDE WDF WDI WDJ WDK WDN WDP WDQ WDR WDT WDU WDV WDW WDZ +mapStations2AHL SACN90 CWAO COLL WEA WEB WEC WEE WEF WEI WEL WEO WEP WER WEW WEZ WFE WFF WFJ WFO WFP WFQ WFW WFX WFZ WGB WGD WGF WGH WGL WGM WGN +mapStations2AHL SACN91 CWAO COLL WGR WGT WGW WGX WGY WHD WHI WHL WHQ WHT WHV WID WIG WII WIJ WIN WIO WIP WIS WIT WIW WIX WIZ WJB WJC WJH +mapStations2AHL SACN92 CWAO COLL VKU WJI WJK WJL WJM WJN WJO WJR WJT WJU WJV WJW WJX WKA WKB WKD WKF WKG WKH WKX WLC WLE WLF WLG WLJ WLK WLM WLQ WLU WLV WME WMF WMI WMJ WMM WMQ +mapStations2AHL SACN93 CWAO COLL GCL MJN WMU WMX WMY WMZ WNC WND WNE WNH WNK WNM WNP WNQ WNR WNT WNV WNX WNZ WOC WOD WOE WOI WOJ WOK WON WOQ WOV WOW WPA WPD WPE WPF WPG WPL WPR WPZ WQA +mapStations2AHL SACN94 CWAO COLL WQC WQD WQE WQF WQH WQK WQL WQM WQO WQP WQQ WQR WQS WQU WQV WQW WQY WQZ WRA WRD WRE WRJ WRK WRM WRR WRT WRU WRV WRY WRZ WSB WSD WSF WSG WSK WSL WSN WSP WSQ WSR WST WSU WSV WSX WSY +mapStations2AHL SACN95 CWAO COLL WTA WTD WTE WTG WTM WTN WTR WTS WTT WTX WTY WUI WUK WUS WUT WUX WUY WVC WVF WVI WVN WVO WVP WVQ WVT WVU WVW WVZ WWA WWB WWC WWE WWF WWK WWN WWO WWP WXA WXB WXC WXG WZW +mapStations2AHL SACN96 CWAO COLL WXL WXN WXP WXS WXW WYA WYE WYF WYH WYI WYJ WYL WYQ WYW WYY WZA WZE WZL WZO WZQ WZR WZS WZT WZV WZY XBD XBQ XEL XFV XGM XHM XKA XKE XND XOA XPS XPT XWF ZGH ZOC ZPK ZQT ZUB ZWN +mapStations2AHL SACN97 CWAO COLL XAT XBK XBO XCA XCH XCK XDE XDI XDW XEA XEC XEG XET XFA XGH XHA XHF XHI XIB XLC XLL XLT XMD XMI XMM XMP XMW XMY XNM XOX XQT XRH XSH XSR XTL XTN XTP XTV XVG XWN +mapStations2AHL SACN98 CWAO COLL XAF XAG XAJ XBA XBR XBW XCD XCP XCS XDK XDP XFR XHR XKI XKM XLB XOY XPA XPC XRB XRL XSE XTD XTO XVN XWB XWM XYH XZV ZKD ZLW ZMJ ZPR ZQZ ZRP ZTB ZZJ ZZP +mapStations2AHL SICN00 CWAO2 71071 +mapStations2AHL SICN00 CWAO1 69050 71324 71325 71326 71327 71328 71329 71330 +mapStations2AHL SICN00 CWAO 71028 71212 71393 71667 71699 71743 71805 71806 71988 71WGX +mapStations2AHL SICN01 CWAO COLL 71043 71045 71050 71051 71052 71058 71066 71068 71076 71077 71079 71082 71083 71090 71091 71092 71093 71094 71095 71101 71104 71107 71108 71109 71729 +mapStations2AHL SICN02 CWAO COLL 71114 71120 71122 71123 71125 71129 71131 71135 71137 71140 71145 71148 71185 71188 71200 71207 71260 +mapStations2AHL SICN03 CWAO COLL 71397 71400 71403 71411 71412 71421 71430 71431 71432 71433 71435 71441 71443 71444 71447 71467 71472 71474 71510 71538 71600 71601 +mapStations2AHL SICN04 CWAO COLL 71603 71607 71609 71610 71621 71623 71624 71625 71627 71628 71630 71631 71633 71705 71706 71707 71709 71714 71715 71718 71719 +mapStations2AHL SICN05 CWAO COLL 71725 71726 71727 71728 71730 71731 71735 71738 71739 71799 71800 71801 71803 71807 71808 71809 71810 71811 71813 71814 71815 +mapStations2AHL SICN06 CWAO COLL 71816 71818 71821 71825 71827 71828 71831 71832 71834 71841 71842 71845 71850 71854 71856 71858 71861 71862 73075 +mapStations2AHL SICN07 CWAO COLL 71864 71865 71867 71868 71869 71871 71877 71878 71880 71882 71883 71887 71889 71892 +mapStations2AHL SICN08 CWAO COLL 71893 71894 71896 71897 71898 71899 71902 71906 71907 71909 71911 71912 71915 71917 71922 71924 71925 71926 71928 +mapStations2AHL SICN09 CWAO COLL 71930 71931 71934 71935 71936 71937 71938 71940 71943 71945 71946 71948 71949 71951 71964 71966 71968 +mapStations2AHL SICN20 CWAO COLL 71001 71002 71003 71005 71006 71007 71008 71009 71010 71011 71012 71013 71014 71015 71016 71017 71018 71020 71021 71022 71023 71024 71026 71027 71028 71029 71030 71031 71032 71033 71035 71036 71038 71039 71040 71041 71042 71044 71046 71047 71048 71049 71060 71063 73001 73002 73005 73011 73015 73016 73017 73018 73024 73025 73026 73027 73029 73030 73070 +mapStations2AHL SICN21 CTST 71037 71119 71338 71591 71676 71677 71798 +mapStations2AHL SICN21 CWAO COLL 71102 71105 71106 71107 71110 71111 71112 71113 71115 71116 71118 71121 71127 71130 71132 71133 71134 71136 71139 71141 71142 71144 71147 71149 73116 +mapStations2AHL SICN22 CWAO COLL 71201 71202 71203 71204 71208 71209 71211 71212 71213 71214 71215 71216 71217 71218 71219 71220 71221 71222 71223 71224 71225 71226 71227 71228 71229 71230 71231 71232 71233 71234 71235 71236 71237 71238 71239 71240 71242 71243 71244 71245 71246 71247 71248 71249 +mapStations2AHL SICN23 CWAO COLL 71300 71301 71302 71303 71304 71305 71306 71307 71308 71309 71310 71311 71312 71314 71315 71316 71317 71318 71319 71320 71321 71322 71323 71324 71327 71328 71331 71332 71334 71335 71336 71337 71338 71339 71340 71341 71342 71343 71344 71345 71346 71347 71348 71349 +mapStations2AHL SICN24 CWAO COLL 71401 71402 71404 71405 71406 71407 71408 71409 71410 71413 71414 71415 71417 71418 71419 71423 71424 71425 71426 71427 71428 71429 71434 71436 71437 71438 71439 71440 71442 71445 71446 71448 71449 +mapStations2AHL SICN25 CWAO COLL 71500 71501 71502 71503 71504 71505 71506 71507 71508 71509 71511 71512 71513 71514 71515 71516 71517 71518 71519 71520 71521 71522 71523 71524 71525 71526 71527 71528 71532 71533 71534 71535 71536 71537 71539 71540 71541 71542 71543 71544 71545 71546 71547 71548 71549 +mapStations2AHL SICN26 CWAO COLL 71602 71604 71605 71606 71608 71611 71612 71613 71614 71615 71616 71617 71618 71619 71622 71629 71632 71634 71635 71636 71637 71638 71640 71641 71642 71644 71645 71646 +mapStations2AHL SICN27 CWAO COLL 71700 71701 71702 71704 71710 71712 71713 71716 71717 71720 71721 71723 71724 71732 71733 71734 71736 71737 71741 71742 71743 71744 71745 71746 71747 71748 +mapStations2AHL SICN28 CWAO COLL 71812 71819 71820 71823 71824 71826 71829 71833 71835 71837 71839 71840 71846 71847 71848 71849 +mapStations2AHL SICN29 CWAO COLL 71900 71903 71904 71905 71914 71916 71918 71920 71921 71923 71927 71932 71933 71941 71942 71944 71947 +mapStations2AHL SICN30 CWAO COLL 71051 71055 71056 71057 71059 71061 71062 71064 71065 71067 71070 71072 71073 71074 71075 71078 71084 71085 71087 71088 71089 71096 71097 71098 71099 73071 73080 +mapStations2AHL SICN31 CWAO COLL 71150 71151 71152 71153 71154 71155 71156 71157 71158 71159 71160 71161 71162 71163 71164 71165 71166 71167 71168 71170 71171 71172 71174 71175 71176 71177 71178 71179 71180 71181 71183 71186 71187 71189 71190 71191 71192 71193 71195 71196 71197 71198 71199 +mapStations2AHL SICN32 CWAO COLL 71250 71251 71252 71253 71254 71255 71257 71258 71259 71261 71264 71263 71265 71266 71267 71268 71269 71270 71271 71272 71273 71274 71275 71276 71277 71278 71279 71281 71282 71283 71284 71285 71286 71287 71288 71289 71290 71291 71292 71293 71294 71295 71296 71297 71298 71299 +mapStations2AHL SICN33 CWAO COLL 71350 71351 71352 71353 71354 71355 71356 71357 71358 71359 71360 71361 71362 71363 71364 71365 71366 71367 71368 71369 71370 71371 71372 71373 71374 71375 71376 71377 71378 71379 71380 71381 71382 71383 71384 71385 71386 71387 71388 71389 71390 71391 71392 71393 71395 71398 71399 +mapStations2AHL SICN34 CWAO COLL 71488 71450 71451 71452 71453 71454 71455 71456 71457 71458 71459 71460 71461 71462 71463 71464 71465 71466 71468 71470 71471 71473 71475 71476 71477 71478 71479 71480 71481 71482 71483 71484 71485 71486 71487 71489 71490 71491 71492 71493 71494 71495 71496 71497 71498 71499 +mapStations2AHL SICN35 CWAO COLL 71550 71551 71552 71553 71554 71555 71556 71557 71558 71559 71560 71564 71565 71566 71567 71568 71569 71570 71571 71572 71573 71574 71575 71576 71577 71578 71579 71580 71581 71582 71584 71585 71586 71587 71588 71589 71590 71592 71594 71595 71596 71597 71598 71599 +mapStations2AHL SICN36 CWAO COLL 71652 71653 71654 71655 71656 71657 71658 71659 71660 71663 71664 71665 71666 71667 71668 71669 71670 71671 71672 71673 71674 71675 71678 71679 71680 71681 71682 71683 71684 71685 71686 71687 71688 71689 71690 71691 71692 71693 71694 71695 71696 71697 71698 71699 +mapStations2AHL SICN37 CWAO COLL 71750 71751 71752 71753 71754 71756 71758 71760 71761 71762 71763 71765 71766 71767 71768 71769 71770 71771 71772 71773 71774 71775 71776 71777 71778 71779 71780 71781 71782 71783 71784 71785 71786 71787 71788 71789 71791 71792 71793 71794 71795 71796 71797 +mapStations2AHL SICN38 CWAO COLL 71851 71855 71857 71859 71872 71874 71876 71884 71885 71886 71888 71890 71895 +mapStations2AHL SICN39 CWAO COLL 71950 71952 71956 71954 71959 71960 71962 71963 71965 71967 71970 71971 71972 71973 71974 71975 71976 71977 71978 71979 71980 71981 71982 71983 71984 71985 71986 71987 71989 71990 71999 +mapStations2AHL SMCN00 CWAO2 71071 +mapStations2AHL SMCN00 CWAO1 69050 71324 71325 71326 71327 71328 71329 71330 +mapStations2AHL SMCN00 CWAO 71699 71805 71806 71988 71WGX 75010 75011 75012 75013 75014 75015 75016 75017 75018 75019 +mapStations2AHL SMCN01 CWAO COLL 71043 71045 71050 71051 71052 71066 71068 71076 71077 71079 71082 71083 71090 71091 71092 71093 71094 71095 71101 71104 71108 71109 71729 +mapStations2AHL SMCN02 CWAO COLL 71114 71120 71122 71123 71125 71129 71131 71135 71137 71140 71145 71148 71185 71188 71200 71207 71260 +mapStations2AHL SMCN03 CWAO COLL 71397 71400 71403 71411 71412 71421 71430 71431 71432 71433 71435 71441 71443 71444 71447 71467 71472 71474 71510 71538 71600 71601 +mapStations2AHL SMCN04 CWAO COLL 71603 71607 71609 71610 71621 71623 71624 71625 71627 71628 71630 71631 71633 71705 71706 71707 71709 71714 71715 71718 71719 +mapStations2AHL SMCN05 CWAO COLL 71725 71726 71727 71728 71730 71731 71735 71738 71739 71799 71800 71801 71803 71807 71808 71809 71810 71811 71813 71814 71815 +mapStations2AHL SMCN06 CWAO COLL 71816 71818 71821 71825 71827 71828 71831 71832 71834 71841 71842 71845 71850 71854 71856 71858 71861 71862 73075 +mapStations2AHL SMCN07 CWAO COLL 71864 71865 71867 71868 71869 71871 71877 71878 71880 71882 71883 71887 71889 71892 +mapStations2AHL SMCN08 CWAO COLL 71893 71894 71896 71897 71898 71899 71902 71906 71907 71909 71911 71912 71915 71917 71922 71924 71925 71926 71928 +mapStations2AHL SMCN09 CWAO COLL 71930 71931 71934 71935 71936 71937 71938 71940 71943 71945 71946 71948 71949 71951 71964 71966 71968 +mapStations2AHL SMCN20 CWAO COLL 71001 71002 71003 71005 71006 71007 71008 71009 71010 71011 71012 71013 71014 71015 71016 71017 71018 71020 71021 71022 71023 71024 71026 71027 71028 71029 71030 71031 71032 71033 71035 71036 71038 71039 71040 71041 71042 71044 71046 71047 71048 71049 71060 71063 73001 73002 73005 73011 73015 73016 73017 73018 73023 73024 73025 73026 73027 73029 73030 73070 +mapStations2AHL SMCN21 CTST 71037 71119 71338 71591 71676 71677 71798 +mapStations2AHL SMCN21 CWAO COLL 71102 71105 71106 71107 71110 71111 71112 71113 71115 71116 71118 71121 71127 71130 71132 71133 71134 71136 71139 71141 71142 71144 71147 71149 73116 +mapStations2AHL SMCN22 CWAO COLL 71201 71202 71203 71204 71208 71209 71211 71212 71213 71214 71215 71216 71217 71218 71219 71220 71221 71222 71223 71224 71225 71226 71227 71228 71229 71230 71231 71232 71233 71234 71235 71236 71237 71238 71239 71240 71242 71243 71244 71245 71246 71247 71248 71249 +mapStations2AHL SMCN23 CWAO COLL 71300 71301 71302 71303 71304 71305 71306 71307 71308 71309 71310 71311 71312 71314 71315 71316 71317 71318 71319 71320 71321 71322 71323 71324 71327 71328 71331 71332 71334 71335 71336 71337 71338 71339 71340 71341 71342 71343 71344 71345 71346 71347 71348 71349 +mapStations2AHL SMCN24 CWAO COLL 71401 71402 71404 71405 71406 71407 71408 71409 71410 71413 71414 71415 71417 71418 71419 71423 71424 71425 71426 71427 71428 71429 71434 71436 71437 71438 71439 71440 71442 71445 71446 71448 71449 +mapStations2AHL SMCN25 CWAO COLL 71500 71501 71502 71503 71504 71505 71506 71507 71508 71509 71511 71512 71513 71514 71515 71516 71517 71518 71519 71520 71521 71522 71523 71524 71525 71526 71527 71528 71532 71533 71534 71535 71536 71537 71539 71540 71541 71542 71543 71544 71545 71546 71547 71548 71549 +mapStations2AHL SMCN26 CWAO COLL 71602 71604 71605 71606 71608 71611 71612 71613 71614 71615 71616 71617 71618 71619 71622 71629 71632 71634 71635 71636 71637 71638 71639 71640 71641 71642 71643 71644 71645 71646 +mapStations2AHL SMCN27 CWAO COLL 71700 71701 71702 71704 71710 71712 71713 71716 71717 71720 71721 71723 71724 71732 71733 71734 71736 71737 71741 71742 71743 71744 71745 71746 71747 71748 +mapStations2AHL SMCN28 CWAO COLL 71812 71819 71820 71823 71824 71826 71829 71833 71835 71837 71839 71840 71846 71847 71848 71849 +mapStations2AHL SMCN29 CWAO COLL 71900 71903 71904 71905 71914 71916 71918 71920 71921 71923 71927 71932 71933 71941 71942 71944 71947 +mapStations2AHL SMCN30 CWAO COLL 71051 71055 71056 71057 71059 71061 71062 71064 71065 71067 71070 71072 71073 71074 71075 71078 71084 71085 71087 71088 71089 71096 71097 71098 71099 73071 73080 +mapStations2AHL SMCN31 CWAO COLL 71150 71151 71152 71153 71154 71155 71156 71157 71158 71159 71160 71161 71162 71163 71164 71165 71166 71167 71168 71170 71171 71172 71174 71175 71176 71177 71178 71179 71180 71181 71183 71186 71187 71189 71190 71191 71192 71193 71195 71196 71197 71198 71199 +mapStations2AHL SMCN32 CWAO COLL 71250 71251 71252 71253 71254 71255 71257 71258 71259 71261 71263 71264 71265 71266 71267 71268 71269 71270 71271 71272 71273 71274 71275 71276 71277 71278 71279 71281 71282 71283 71284 71285 71286 71287 71288 71289 71290 71291 71292 71293 71294 71295 71296 71297 71298 71299 +mapStations2AHL SMCN33 CWAO COLL 71350 71351 71352 71353 71354 71355 71356 71357 71358 71359 71360 71361 71362 71363 71364 71365 71366 71367 71368 71369 71370 71371 71372 71373 71374 71375 71376 71377 71378 71379 71380 71381 71382 71383 71384 71385 71386 71387 71388 71389 71390 71391 71392 71393 71395 71398 71399 +mapStations2AHL SMCN34 CWAO COLL 71488 71450 71451 71452 71453 71454 71455 71456 71457 71458 71459 71460 71461 71462 71463 71464 71465 71466 71468 71470 71471 71473 71475 71476 71477 71478 71479 71480 71481 71482 71483 71484 71485 71486 71487 71489 71490 71491 71492 71493 71494 71495 71496 71497 71498 71499 +mapStations2AHL SMCN35 CWAO COLL 71550 71551 71552 71553 71554 71555 71556 71557 71558 71559 71560 71564 71565 71566 71567 71568 71569 71570 71571 71572 71573 71574 71575 71576 71577 71578 71579 71580 71581 71582 71584 71585 71586 71587 71588 71589 71590 71592 71594 71595 71596 71597 71598 71599 +mapStations2AHL SMCN36 CWAO COLL 71652 71653 71654 71655 71656 71657 71658 71659 71660 71663 71664 71665 71666 71667 71668 71669 71670 71671 71672 71673 71674 71675 71678 71679 71680 71681 71682 71683 71684 71685 71686 71687 71688 71689 71690 71691 71692 71693 71694 71695 71696 71697 71698 71699 +mapStations2AHL SMCN37 CWAO COLL 71744 71750 71751 71752 71753 71754 71756 71758 71760 71761 71762 71763 71765 71766 71767 71768 71769 71770 71771 71772 71773 71774 71775 71776 71777 71778 71779 71780 71781 71782 71783 71784 71785 71786 71787 71788 71789 71791 71792 71793 71794 71795 71796 71797 +mapStations2AHL SMCN38 CWAO COLL 71851 71855 71857 71859 71872 71874 71876 71884 71885 71886 71888 71890 71895 +mapStations2AHL SMCN39 CWAO COLL 71950 71952 71954 71956 71959 71960 71962 71963 71965 71967 71970 71971 71972 71973 71974 71975 71976 71977 71978 71979 71980 71981 71982 71983 71984 71985 71986 71987 71988 71989 71990 71999 +mapStations2AHL UECN01 CWAO 71600 71603 71701 71802 71815 71816 71843 73033 +mapStations2AHL UECN02 CWAO 71625 71716 71722 71811 71823 71836 71845 71906 71907 +mapStations2AHL UECN03 CWAO 71119 71853 71867 71876 71913 +mapStations2AHL UECN04 CWAO 71081 71082 71826 71909 71915 71917 71924 +mapStations2AHL UECN05 CWAO 71043 71925 71926 71934 71957 +mapStations2AHL UECN06 CWAO 71109 71203 71908 71945 71964 +mapStations2AHL UECN10 CWAO 71561 71569 +mapStations2AHL UECN21 CTST 71126 71156 71396 71591 +mapStations2AHL UECN21 CWAO 71086 71124 71145 71248 71359 71360 71371 71394 71486 71488 71621 71626 71712 71714 71877 71893 CGDT +mapStations2AHL UECN22 CWAO CEUC CEUQ 71054 CEWA 71194 CEWM 71146 CEWN CEWO 71117 CEWP 71000 CEWT CEWW 71169 CEWY +mapStations2AHL UFCN01 CWBZ 71712 +mapStations2AHL UFCN21 CTST 71591 +mapStations2AHL UGCN01 CWAO 71600 71603 71701 71802 71815 71816 71843 73033 +mapStations2AHL UGCN02 CWAO 71625 71716 71722 71811 71823 71836 71845 71906 71907 +mapStations2AHL UGCN03 CWAO 71119 71120 71129 71853 71867 71876 71913 +mapStations2AHL UGCN04 CWAO 71081 71082 71826 71909 71915 71917 71924 +mapStations2AHL UGCN05 CWAO 71043 71925 71926 71934 71957 +mapStations2AHL UGCN06 CWAO 71109 71203 71908 71945 71964 +mapStations2AHL UGCN10 CWAO 71561 71569 +mapStations2AHL UGCN21 CTST 71126 71156 71396 71591 +mapStations2AHL UGCN21 CWAO 71086 71124 71145 71248 71262 71359 71360 71371 71394 71486 71488 71621 71712 71714 71877 71893 CGDT +mapStations2AHL UGCN22 CWAO CEUC CEUQ 71054 CEWA 71194 CEWM 71146 CEWN CEWO 71117 CEWP 71000 CEWT CEWW 71169 CEWY +mapStations2AHL UGCN90 CWAO 99910 +mapStations2AHL UHCN01 CWAO 71120 71121 71129 71621 71852 +mapStations2AHL UHCN21 CTST 71054 71126 +mapStations2AHL UHCN21 CWAO 71248 71359 71360 71394 71712 71877 CGDT +mapStations2AHL UHCN22 CWAO CEUC CEUQ CEWA CEWM CEWN CEWO CEWP CEWT CEWW CEWY +mapStations2AHL UKCN01 CWAO 71600 71603 71701 71802 71815 71816 71843 73033 +mapStations2AHL UKCN02 CWAO 71625 71716 71722 71811 71823 71836 71845 71906 71907 +mapStations2AHL UKCN03 CWAO 71119 71853 71867 71876 71913 +mapStations2AHL UKCN04 CWAO 71081 71082 71826 71909 71915 71917 71924 +mapStations2AHL UKCN05 CWAO 71043 71925 71926 71934 71957 +mapStations2AHL UKCN06 CWAO 71109 71203 71908 71945 71964 +mapStations2AHL UKCN10 CWAO 71561 71569 +mapStations2AHL UKCN21 CTST 71126 71156 71396 71591 +mapStations2AHL UKCN21 CWAO 71086 71124 71145 71248 71359 71360 71371 71394 71486 71488 71621 71626 71712 71714 71877 71893 CGDT +mapStations2AHL UKCN22 CWAO CEUC CEUQ 71054 CEWA 71194 CEWM 71146 CEWN CEWO 71117 CEWP 71000 CEWT CEWW 71169 CEWY +mapStations2AHL UKCN90 CWAO 69150 99910 +mapStations2AHL ULCN01 CWAO 71600 71603 71701 71802 71815 71816 73033 +mapStations2AHL ULCN01 CYOY 71716 71843 +mapStations2AHL ULCN02 CWAO 71625 71716 71722 71811 71823 71836 71845 71906 71907 +mapStations2AHL ULCN03 CWAO 71119 71853 71867 71876 71913 +mapStations2AHL ULCN04 CWAO 71081 71082 71826 71909 71915 71917 71924 +mapStations2AHL ULCN05 CWAO 71043 71925 71926 71934 71957 +mapStations2AHL ULCN06 CWAO 71109 71203 71908 71945 71964 +mapStations2AHL ULCN10 CWAO 71561 71569 +mapStations2AHL ULCN21 CTST 71126 71156 71396 71591 +mapStations2AHL ULCN21 CWAO 71086 71124 71145 71248 71359 71360 71371 71394 71486 71488 71621 71626 71712 71714 71877 71893 CGDT +mapStations2AHL ULCN22 CWAO CEUC CEUQ 71054 CEWA 71194 CEWM 71146 CEWN CEWO 71117 CEWP 71000 CEWT CEWW 71169 CEWY +mapStations2AHL UPCN01 CWAO 71120 71121 71129 71621 71852 +mapStations2AHL UPCN20 CWAO 71145 +mapStations2AHL UPCN21 CTST 71054 71126 +mapStations2AHL UPCN21 CWAO 71248 71359 71360 71394 71712 71877 CGDT +mapStations2AHL UPCN22 CWAO CEUC CEUQ CEWA CEWM CEWN CEWO CEWP CEWT CEWW CEWY +mapStations2AHL UQCN01 CWAO 71600 71603 71701 71802 71815 71816 71843 73033 +mapStations2AHL UQCN02 CWAO 71625 71716 71722 71811 71823 71836 71845 71906 71907 +mapStations2AHL UQCN03 CWAO 71119 71120 71129 71853 71867 71876 71913 +mapStations2AHL UQCN04 CWAO 71081 71082 71826 71909 71915 71917 71924 +mapStations2AHL UQCN05 CWAO 71043 71925 71926 71934 71957 +mapStations2AHL UQCN06 CWAO 71109 71203 71908 71945 71964 +mapStations2AHL UQCN10 CWAO 71561 71569 +mapStations2AHL UQCN21 CTST 71126 71156 71396 71591 +mapStations2AHL UQCN21 CWAO 71086 71124 71145 71248 71359 71360 71371 71394 71486 71488 71712 71714 71877 71893 CGDT +mapStations2AHL UQCN22 CWAO CEUC CEUQ 71054 CEWA 71194 CEWM 71146 CEWN CEWO 71117 CEWP 71000 CEWT CEWW 71169 CEWY +mapStations2AHL USCN01 CWAO 71600 71603 71701 71802 71815 71816 71843 73033 +mapStations2AHL USCN02 CWAO 71625 71716 71722 71811 71823 71836 71845 71906 71907 +mapStations2AHL USCN03 CWAO 71119 71853 71867 71876 71913 +mapStations2AHL USCN04 CWAO 71081 71082 71826 71909 71915 71917 71924 +mapStations2AHL USCN05 CWAO 71043 71925 71926 71934 71957 +mapStations2AHL USCN06 CWAO 71109 71203 71908 71945 71964 +mapStations2AHL USCN10 CWAO 71561 71569 +mapStations2AHL USCN21 CTST 71126 71156 71396 71591 +mapStations2AHL USCN21 CWAO 71086 71124 71145 71248 71262 71359 71360 71371 71394 71486 71488 71621 71626 71712 71714 71877 71893 CGDT +mapStations2AHL USCN22 CWAO CEUC CEUQ 71054 CEWA 71194 CEWM 71146 CEWN CEWO 71117 CEWP 71000 CEWT CEWW 71169 CEWY +mapStations2AHL USCN90 CWAO 69150 99910 +mapStations2AHL USCN99 CWAO CEWA CNEWA diff --git a/am_flow/config/sarra/get_from-watch_f02.conf b/am_flow/config/sarra/get_from-watch_f02.conf new file mode 100644 index 0000000..19ef0ff --- /dev/null +++ b/am_flow/config/sarra/get_from-watch_f02.conf @@ -0,0 +1,52 @@ +# source + +instances 1 + +# broker is localhost and all products are processed locally + +topic_prefix v03.post +broker ${MQP}://tfeed@${FLOWBROKER}/ +exchange xs_tfeed_am + +# listen to spread the poll messages + +prefetch 10 + +# Does not work ATM (2024/01/05). Waiting for R3.00.51 of sr3 to come out +#sourceFromExchange True + +# what to do with product + +mirror True +timeCopy False +delete False + +# extension -> moved to ~/master/sarra/poll/airnow_ab.conf +# header sundew_extension=AQ_MST:AB:AIRNOW:AQCSV: + +# plugin to make the products with unique names + +# RS ACDC 21955 +permCopy False +permDefault 664 + +# directories + +directory ${TESTDOCROOT}/bulletins_to_download/${YYYYMMDD} + +# These bulletins are too long for AM to handle transferring +reject .*FP/.* +# Problematic canadian and american bulletins +reject .*SACN[34].* +reject .*KWAL.* +accept .* +acceptUnmatched False + +# destination + +post_broker ${MQP}://tsource@${FLOWBROKER}/ +post_exchange xs_tsource_am +post_baseUrl file://home/home/sarra/sarra_devdocroot/bulletins_to_download +post_baseDir / + +logLevel debug diff --git a/am_flow/config/sender/am_send_f04.conf b/am_flow/config/sender/am_send_f04.conf new file mode 100644 index 0000000..8a0eaba --- /dev/null +++ b/am_flow/config/sender/am_send_f04.conf @@ -0,0 +1,12 @@ +topic_prefix v03.post + +callback gather.message +callback send.am +broker ${MQP}://tsource@${FLOWBROKER} +exchange xs_tsource_am +accept .* +sum sha512 +MaxBulLen 32768 + +sendTo am://${FLOWBROKER}:5005 +debug on diff --git a/am_flow/config/subscribe/bulletin_subscribe_f05.conf b/am_flow/config/subscribe/bulletin_subscribe_f05.conf new file mode 100644 index 0000000..4227518 --- /dev/null +++ b/am_flow/config/subscribe/bulletin_subscribe_f05.conf @@ -0,0 +1,24 @@ +# download localhost via xpublic exchange (uses http) +# data comes from amserver flow + +broker ${MQP}://tsub@${FLOWBROKER}/ +topic_prefix v03.post +exchange xpublic + +# logReject +messageDebugDump on +logLevel debug + +strip 1 +directory ${TESTDOCROOT}/bulletins_subscribe/ + +msg_total_interval 0 +plugin msg_total_save + +file_total_interval 0 +plugin file_total_save + +mirror True + +accept .* + diff --git a/am_flow/config/watch/watch-dir_f03.conf b/am_flow/config/watch/watch-dir_f03.conf new file mode 100644 index 0000000..70a271f --- /dev/null +++ b/am_flow/config/watch/watch-dir_f03.conf @@ -0,0 +1,25 @@ +post_topic_prefix v03.post + +post_broker ${MQP}://tfeed@${FLOWBROKER}/ +post_exchange xs_tfeed_am +post_baseUrl file://home/home/sarra/sarra_devdocroot/bulletins_to_post/bulletins +post_baseDir / + + +path ${TESTDOCROOT}/bulletins_to_post/ + +logLevel info + + +fileEvents create + +#caching False +#recursive True + +strip 5 +reject .*tmp.* +accept .* + +inflight 5 + +debug True diff --git a/am_flow/doc/single_flow.dia b/am_flow/doc/single_flow.dia new file mode 100644 index 0000000..eeda05e Binary files /dev/null and b/am_flow/doc/single_flow.dia differ diff --git a/am_flow/flow_check.sh b/am_flow/flow_check.sh new file mode 100755 index 0000000..8dba482 --- /dev/null +++ b/am_flow/flow_check.sh @@ -0,0 +1,269 @@ +#!/bin/bash + +flow_test_name="`basename pwd`" + +# parse arguments +POSITIONAL=() +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + -s|--skip_summaries) + skip_summaries=true + shift # past argument + ;; +esac +done +set -- "${POSITIONAL[@]}" + +. ./flow_include.sh +countall + +function summarize_performance { + path="$LOGDIR"/$1 + shift + pattern=$1 + shift + for i in $* ; do + best_fn='' + printf "\n\t$i\n\n" + for j in ${path}_${i}_*.log; do + msg="`grep -a ${pattern} ${j} | tail -1`" + if [[ -z "$msg" ]]; then + continue + fi + fn=`echo $(basename ${j}) | awk -F'.' '{print $3}'` + if [[ -z "$fn" ]]; then + best_fn=`echo $(basename ${j})` + echo "`basename $j` ${msg}" + elif [[ -z "$best_fn" ]]; then + echo "`basename $j` ${msg}" + fi + done + done +} + +function summarize_logs { + printf "\n$1 Summary:\n" + input_size=${#1} + fcl="$LOGDIR"/flowcheck_$1_logged.txt + msg_counts=`grep -a -h -o "\[$1\] *.*" "$LOGDIR"/*.log | sort | uniq -c -w"$((input_size+20))" | sort -n -r` + echo '' > ${fcl} + + if [[ -z $msg_counts ]]; then + echo NO $1S IN LOGS + else + backup_ifs=$IFS + IFS=$'\n' + for msg_line in $msg_counts; do + count=`echo ${msg_line} | awk '{print $1}'` + msg=`echo ${msg_line} | sed "s/^ *[0-9]* \[$1\] *//g"` + pattern="\[$1\] *${msg}" + filelist=($(grep -a -l ${pattern::$((input_size + 22))} "$LOGDIR"/*.log)) + if [[ ${filelist[@]} ]]; then + first_filename=`basename ${filelist[0]} | sed 's/ /\n/g' | sed 's|.*\/||g' | sed 's/_[0-9][0-9]\.log\|.log//g' | uniq` + files_nb=${#filelist[@]} + echo " ${count}"$'\u2620'"${first_filename}"$'\u2620'"(${files_nb} file)"$'\u2620'"`echo ${msg_line} | sed "s/^ *[0-9]* //g"`" >> ${fcl} + echo ${filelist[@]} | sed 's/^//g' | sed 's/ \//\n\//g' >> ${fcl} + echo -e >> ${fcl} + fi + done + IFS=${backup_ifs} + result=`grep -a -c $1 ${fcl}` + if [[ ${result} -gt 10 ]]; then + grep -a $1 ${fcl} | head | column -t -s $'\u2620' | cut -c -130 + echo + echo "More than 10 TYPES OF $1S found... for the rest, have a look at $fcl for details" + else + grep -a $1 ${fcl} | column -t -s $'\u2620' | cut -c -130 + fi + fi +} + +function checktree_md5sum { + tree=$1 + printf "checking +${tree}+\n" + SUMDIR=${LOGDIR}/sums + if [ ! -d $SUMDIR ]; then + mkdir $SUMDIR + fi + + report=${SUMDIR}/`basename ${tree}`_md5.txt + #if [ ! -f ${report} ]; then + (cd ${tree}; find . \! -type d | xargs md5sum ) | sort > ${report} + #fi + +} + +function checktree_fn { + tree=$1 + printf "checking +${tree}+\n" + SUMDIR=${LOGDIR}/fn + if [ ! -d $SUMDIR ]; then + mkdir $SUMDIR + fi + + report=${SUMDIR}/`basename ${tree}`_fn.txt + #if [ ! -f ${report} ]; then + (cd ${tree}; find . \! -type d | awk '{print $NF}' | cut -d '/' -f6- | cut -d '_' -f-5 ) | sort > ${report} + #fi + +} + +function logPermCheck { + tno=$((${tno}+1)) + + #looking into the configs for chmod_log commands if they exist + perms="`grep -a chmod_log config -r`" + file1=`grep -a chmod_log config -r | cut -f2 -d"/"` + file2=`grep -a chmod_log config -r | cut -f3 -d"/" | cut -f1 -d"."` + + #finding the log related to the config file + if [ "${sarra_py_version:0:1}" == "3" ]; then + path=$HOME/.cache/sr3/log/"$file1"_*.log + else + path=$HOME/.cache/sarra/log/sr_"$file1"_*.log + fi + #printf "$path \n" + + #checking if the perms from the config is reflected in the file + fileperms=`stat -c "%a %n" $path` + if [[ "$fileperms" == *"${perms: -3}"* ]]; then + printf "test %d success: Log perms confirmed\n" $tno + passedno=$((${passedno}+1)) + else + printf "test %d FAILURE: Log perms test failed.\n" $tno + fi +} + +function comparetree_md5 { + + tno=$((${tno}+1)) + SUMDIR=${LOGDIR}/sums + diff ${SUMDIR}/${1}_md5.txt ${SUMDIR}/${2}_md5.txt >/dev/null 2>&1 + result=$? + + if [ $result -gt 0 ]; then + printf "test %d FAILURE: compare contents of ${1}_md5 and ${2}_md5 differ\n" $tno + else + printf "test %d success: compare contents of ${1}_md5 and ${2}_md5 are the same\n" $tno + passedno=$((${passedno}+1)) + fi + +} + +function comparetree_fn { + + tno=$((${tno}+1)) + SUMDIR=${LOGDIR}/fn + diff ${SUMDIR}/${1}_fn.txt ${SUMDIR}/${2}_fn.txt >/dev/null 2>&1 + result=$? + + if [ $result -gt 0 ]; then + printf "test %d FAILURE: compare contents of ${1}_fn and ${2}_fn differ\n" $tno + else + printf "test %d success: compare contents of ${1}_fn and ${2}_fn are the same\n" $tno + passedno=$((${passedno}+1)) + fi + +} + +printf "checking trees...\n" +# Remove special characters +# for bulletinsdir in bulletins_to_{download,post,send} bulletins_subscribe +# do +# files2sed=$(find ${testdocroot}/$bulletinsdir -type f ) +# for bulletin in $files2sed +# do +# sed -i $'s/^M//g' $bulletin +# done +# done + +checktree_md5sum ${testdocroot}/bulletins_subscribe +checktree_md5sum ${testdocroot}/bulletins_to_send + +checktree_fn ${testdocroot}/bulletins_to_download +checktree_fn ${testdocroot}/bulletins_subscribe +checktree_fn ${testdocroot}/bulletins_to_send + + + + +if [[ -z "$skip_summaries" ]]; then + # PAS performance summaries + printf "\nDownload Performance Summaries:\tLOGDIR=$LOGDIR\n" + summarize_performance ${LGPFX}sarra msg_total: get_from-watch_f02 + summarize_performance ${LGPFX}flow msg_total: amserver-flow_f01 + summarize_performance ${LGPFX}subscribe file_total: bulletin_subscribe_f05 + + echo + echo + + if [[ ! "$SARRA_LIB" ]]; then + echo NB retries for ${LGPFX}sarra `grep -a Retrying "$LOGDIR"/${LGPFX}sarra*.log | wc -l` + echo NB retries for ${LGPFX}flow `grep -a Retrying "$LOGDIR"/${LGPFX}flow*.log | wc -l` + echo NB retries for ${LGPFX}subscribe `grep -a Retrying "$LOGDIR"/${LGPFX}subscribe*.log | wc -l` + # else + # echo NB retries for "$SARRA_LIB"/${LGPFX}subscribe.py amqp_f30 `grep -a Retrying "$LOGDIR"/${LGPFX}subscribe_amqp_f30*.log | wc -l` + # echo NB retries for "$SARRA_LIB"/${LGPFX}sender.py `grep -a Retrying "$LOGDIR"/${LGPFX}sender*.log | wc -l` + fi + + summarize_logs ERROR + summarize_logs WARNING +fi + + +passedno=0 +tno=0 + + +printf "\t\tTEST RESULTS\n\n" + +logPermCheck + +echo " | content of subdirs of ${testdocroot} |" +# We can't compare with the tree that we download from the sarra because the station mappings is missing entries inside of the flow +# So we can't compare the filenames as a lot of them are different. +comparetree_md5 bulletins_to_send bulletins_subscribe +comparetree_fn bulletins_to_send bulletins_subscribe + +echo "broker state:" +if [[ ${messages_unacked} > 0 ]] || [[ ${messages_ready} > 0 ]]; then + + echo "rabbitmq broker message anomalies\n" + rabbitmqadmin -H localhost -u bunnymaster -p ${adminpw} list queues name messages_ready messages_unacknowledged | awk ' BEGIN {t=0; } (NR<3) {print;} (NR > 2) && /_f[0-9][0-9]/ { t+=$4; if ( $4 > 0 || $6 > 0) print; }; ' + +fi + +echo " | Message posting |" +calcres "${totwatch}" "${totsarx}" "${LGPFX}watch\t\t (${totwatch}) should post be the same as the messages filtered by the sarra\t\t (${totsarx})" +calcres "${totsarp}" "${totsentx}" "${LGPFX}sarra\t\t (${totsarp}) should publish the same number of items as ${LGPFX}sender accepts (${totsentx})" +calcres "${totsentx}" "${totflowp}" "sender\t\t (${totsentx}) should post same number of items as ${LGPFX}flow posts (${totflowp})" +echo " | Downloaded files |" +calcres "${totsard}" "${totflowd}" "${LGPFX}sarra\t (${totsard}) should download same number of files as ${LGPFX}flow downloads\t (${totflowd})" +calcres "${totflowd}" "${totsub}" "${LGPFX}flow\t (${totflowd}) should download same number of files as ${LGPFX}subscribe downloads (${totsub})" +zerowanted "${missed_dispositions}" "${maxshovel}" "messages received that we don't know what happened." + +if [ "$MQP" == "amqp" ]; then + zerowanted "${messages_unacked}" "${maxshovel}" "there should be no unacknowledged messages left, but there are ${messages_unacked}" + zerowanted "${messages_ready}" "${maxshovel}" "there should be no messages ready to be consumed but there are ${messages_ready}" +fi + +echo "Overall ${flow_test_name} ${passedno} of ${tno} passed (sample size: $staticfilecount) !" + + +if [ "${passedno}" -gt 0 -a "${passedno}" -eq "${tno}" ]; then + results=0 +else + results=$(( "${tno}"-"${passedno}" )) +fi + +if [[ "${missed_dispositions}" -gt 0 ]]; then + # PAS missed_dispositions means definite Sarra bug, very serious. + echo "Please review $missedreport" + results=1 +fi +echo + +exit ${results} diff --git a/am_flow/flow_cleanup.sh b/am_flow/flow_cleanup.sh new file mode 100755 index 0000000..b6ad887 --- /dev/null +++ b/am_flow/flow_cleanup.sh @@ -0,0 +1,112 @@ +#!/bin/bash + + +export TESTDIR="`pwd`" + +. ../flow_utils.sh + +flowlogcleanup="$LOGDIR/flowcleanup_f99.log" +touch $flowlogcleanup +flow_configs="audit/ `cd $CONFDIR; ls */*f[0-9][0-9].conf 2>/dev/null; ls poll/pulse.conf 2>/dev/null`" +flow_configs="`echo ${flow_configs} | tr '\n' ' '`" + +echo remove x attributes added by post then calculating checksums. in ${SAMPLEDATA} +if [ `find ${SAMPLEDATA} -type f | xargs xattr -l|wc -l` ]; then + find ${SAMPLEDATA} -type f | xargs xattr -d user.sr_mtime >&/dev/null + find ${SAMPLEDATA} -type f | xargs xattr -d user.sr_integrity >&/dev/null + find ${SAMPLEDATA} -type f | xargs xattr -d user.sr_identity >&/dev/null + find ${SAMPLEDATA} -type f | xargs xattr -d user.sr_sum >&/dev/null +fi +echo done with xattr + +# Stopping sr components +sr_action "Stopping sr..." stop " " ">> $flowlogcleanup 2>\\&1" "$flow_configs" +# Cleanup sr components +sr_action "Cleanup sr..." cleanup " " ">> $flowlogcleanup 2>\\&1" "$flow_configs" + +echo "Cleanup trivial http server... " +if [ -f .httpserverpid ]; then + httpserverpid="`cat .httpserverpid`" + if [ "${httpserverpid}" -a "`ps ax | awk ' $1 == '${httpserverpid}' { print $1; }; '`" ]; then + kill $httpserverpid + echo "Web server stopped." + sleep 2 + else + echo "No web server found running from pid file" + fi + + echo "If other web servers with lost pid kill them" + pgrep -al python3 | grep trivialserver.py | grep -v grep | xargs -n1 kill 2> /dev/null + + if [ "`netstat -an | grep LISTEN | grep 8001`" ]; then + pid="`ps ax | grep trivialserver.py | grep -v grep| awk '{print $1;};'`" + echo "Killing rogue web server found on port 8001 at pid=$pid" + if [ "$pid" ]; then + kill -9 $pid + else + echo "ERROR: could not find web server, but it's running. Look out!" + fi + fi +fi + +echo "Cleanup flow_post... " +if [ -f .flowpostpid ]; then + flowpostpid="`cat .flowpostpid`" + if [ "${flowpostpid}" -a "`ps ax | awk ' $1 == '${flowpostpid}' { print $1; }; '`" ]; then + kill $flowpostpid + echo "Flow_post stopped." + sleep 2 + else + echo "No properly started flow_post found running from pid file" + fi + + echo "If other flow_post with lost pid kill them" + pgrep flow_post.sh | grep -v grep | xargs -n1 kill 2> /dev/null + +fi + +# This where we start cleaning the cache +remove_if_present=".ftpserverpid .httpserverpid aaa.conf bbb.inc checksum_AHAH.py sr_http.test.anonymous $exnow $missedreport $srposterlog $trivialhttplog" +rm -f ${remove_if_present} + +queues_to_delete="`rabbitmqadmin -H localhost -u bunnymaster -p ${adminpw} -f tsv list queues | awk ' ( NR > 1 ) && /\.sr_.*_f[0-9][0-9].*/ { print $1; }; '`" + +echo "Deleting queues: $queues_to_delete" +for q in $queues_to_delete; do + rabbitmqadmin -H localhost -u bunnymaster -p "${adminpw}" delete queue name=$q >>$flowlogcleanup 2>&1 +done + +exchanges_to_delete="`rabbitmqadmin -H localhost -u bunnymaster -p ${adminpw} -f tsv list exchanges | awk ' ( $1 ~ /x.*/ ) { print $1; }; '`" +echo "Deleting exchanges..." +for exchange in $exchanges_to_delete ; do + rabbitmqadmin -H localhost -u bunnymaster -p ${adminpw} -f tsv delete exchange name=${exchange} >>$flowlogcleanup 2>&1 +done + +flow_configs="`cd ${SR_TEST_CONFIGS}; ls */*f[0-9][0-9].conf 2>/dev/null; ls */*f[0-9][0-9].inc 2>/dev/null; ls poll/pulse.conf 2>/dev/null`" +flow_configs="`echo ${flow_configs} | tr '\n' ' '`" + +sr_action "Removing flow configs..." remove " " ">> $flowlogcleanup 2>\\&1" "$flow_configs" + +echo "Removing flow config logs..." +if [ "$1" != "skipconfig" ]; then + if [ "${sarra_py_version:0:1}" == "3" ]; then + echo $flow_configs | sed 's/ / ;\n rm -f /g' | sed '1 s|^| rm -f |' | sed '/^ rm -f post/d' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].log\*/g' | (cd $LOGDIR; sh ) + echo $flow_configs | sed 's/ / ;\n rm -f /g' | sed '1 s|^| rm -f |' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].json\*/g'| (cd ${LOGDIR}/../metrics; sh ) + else + echo $flow_configs | sed 's/ / ;\n rm -f sr_/g' | sed '1 s|^| rm -f sr_|' | sed '/^ rm -f sr_post/d' | sed 's+/+_+g' | sed '/conf[ ;]*$/!d' | sed 's/\.conf/_[0-9][0-9].log\*/g' | (cd $LOGDIR; sh ) + fi +fi + +echo "Removing flow cache/state files ..." +echo $flow_configs | sed 's/ / ; rm $CACHEDIR\//g' | sed 's/^/rm $CACHEDIR\//' | sed 's+\.conf+/*+g' | sh - 2>/dev/null +echo "$CACHEDIR/*_unit_test" | sed 's/ / ; rm -rf /g' | sed 's/^/rm -rf /' | sh + +httpdr="" +if [ -f .httpdocroot ]; then + httpdr="`cat .httpdocroot`" + if [ "$httpdr" -a -d "$httpdr" ]; then + echo "Removing document root ( $httpdr )..." + rm -rf $httpdr + fi +fi +echo "Done!" diff --git a/am_flow/flow_include.sh b/am_flow/flow_include.sh new file mode 100755 index 0000000..e51392d --- /dev/null +++ b/am_flow/flow_include.sh @@ -0,0 +1,243 @@ +#!/bin/bash + +. ../flow_utils.sh + +C_ALSO="`which sr_cpost`" + +if [ ! "${C_ALSO}" ]; then + C_ALSO="`which sr3_cpost`" +fi + +# The directory we run the flow test scripts in... +tstdir="`pwd`" +httpdocroot=`cat $tstdir/.httpdocroot` +testdocroot="$HOME/sarra_devdocroot" + +function countthem { + if [ ! "${1}" ]; then + tot=0 + else + tot="${1}" + fi +} + +function chkargs { + + if [[ ! "${1}" || ! "${2}" ]]; then + printf "test %2d FAILURE: blank results! ${3}\n" ${tno} + return 2 + fi + if [[ "${1}" -eq 0 ]]; then + printf "test %2d FAILURE: no successful results! ${3}\n" ${tno} + return 2 + fi + + if [[ "${2}" -eq 0 ]]; then + printf "test %2d FAILURE: no successful results, 2nd item! ${3}\n" ${tno} + return 2 + fi + + return 0 +} + +function calcres { + # + # calcres - Calculate test result. + # + # logic: + # increment test number (tno) + # compare first and second totals, and report agreement if within some percent of one another. + # see maxerr calculation... + # emit description based on agreement. Arguments: + # 1 - first total + # 2 - second total + # 3 - test description string. + # 4 - will retry flag. + # + + tno=$((${tno}+1)) + + chkargs "${1}" "${2}" "${3}" + if [ $? -ne 0 ]; then + return $? + fi + + res=0 + + mean=$(( (${1} + ${2}) / 2 )) + maxerr=$(( $mean / 1000 )) + + min=$(( $mean - $maxerr )) + max=$(( $mean + $maxerr )) + + if [ $1 -lt $min -o $2 -lt $min -o $1 -gt $max -o $1 -gt $max ]; then + printf "test %2d FAILURE: ${3}\n" ${tno} + if [ "$4" ]; then + tno=$((${tno}-1)) + fi + return 1 + else + printf "test %2d success: ${3}\n" ${tno} + passedno=$((${passedno}+1)) + return 0 + fi + +} + +function tallyres { + # tallyres - All the results must be good. 99/100 is bad! + # + # logic: + # increment test number (tno) + # compare first and second totals, and report agreement of one another. + # emit description based on agreement. Arguments: + # 1 - value obtained + # 2 - value expected + # 3 - test description string. + + tno=$((${tno}+1)) + + if [ ${1} -ne ${2} -o ${2} -eq 0 ]; then + printf "test %2d FAILURE: ${3}\n" ${tno} + if [ "$4" ]; then + tno=$((${tno}-1)) + fi + return 1 + else + printf "test %2d success: ${3}\n" ${tno} + passedno=$((${passedno}+1)) + return 0 + fi + +} + +function zerowanted { + # zerowanted - this value must be zero... checking for bad things. + # + # logic: + # increment test number (tno) + # compare first and second totals, and report agreement if within 10% of one another. + # emit description based on agreement. Arguments: + # 1 - value obtained + # 2 - samplesize + # 3 - test description string. + + tno=$((${tno}+1)) + + if [ "${2}" -eq 0 ]; then + printf "test %2d FAILURE: no data! ${3}\n" ${tno} + return + fi + + if [ "${1}" -gt 0 ]; then + printf "test %2d FAILURE: ${1} ${3}\n" ${tno} + else + printf "test %2d success: ${1} ${3}\n" ${tno} + passedno=$((${passedno}+1)) + fi +} + +function sumlogs { + + pat="$1" + shift + tot=0 + for l in $*; do + to_add="`grep -a "\[INFO\] $pat" $l | tail -1 | awk ' { print $5; }; '`" + if [ "$to_add" ]; then + tot=$((${tot}+${to_add})) + fi + done +} + +function sumlogshistory { + p="$1" + shift + if [[ $(ls $* 2>/dev/null) ]]; then + reverse_date_logs=`ls $* | sort -n -r` + + for l in $reverse_date_logs; do + if [[ ${tot} = 0 ]]; then + sumlogs $p $l + fi + done + fi +} + + +function countall { + + if [ "${sarra_py_version:0:1}" == "3" ]; then + countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}watch_watch-dir_f03_*.log | grep -v directory | wc -l`" + fi + totfilewatch="${tot}" + + if [[ "${sarra_py_version}" > "3.00.25" ]]; then + countthem "`grep -a 'after_work directory ok' ${LOGDIR}/${LGPFX}watch_watch-dir_f03_*.log | awk ' { print $8; } ' | sort -u | wc -l`" + totdirwatch="${tot}" + else + echo "sr3 version too old to produce directory posts" + totdirwatch=0 + fi + + totwatch=$((${totfilewatch}+${totdirwatch})) + + if [ "${sarra_py_version:0:1}" == "3" ]; then + countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}flow_amserver-flow_f01_*.log | grep -v directory | wc -l`" + totflowp="${tot}" + fi + + if [[ "${sarra_py_version}" > "3.00.53" ]]; then + countthem "`grep -a 'after_work flowed ok' ${LOGDIR}/${LGPFX}flow_amserver-flow_f01_*.log | awk ' { print $8; } ' | sort -u | wc -l`" + totflowd="${tot}" + else + echo "sr3 version too old to produce directory posts" + totdirflow=0 + fi + + + if [[ "${sarra_py_version}" > "3.00.25" ]]; then + countthem "`grep -a 'after_accept accepted' "$LOGDIR"/${LGPFX}sender_am_send_f04_*.log | wc -l`" + totsentx="${tot}" + + fi + + totfilesent="${tot}" + # totsent=$((${totfilesent}+${totdirsent})) + + #countthem "`grep -a '\[INFO\] post_log notice' "$LOGDIR"/${LGPFX}sarra_download_f20_*.log | wc -l`" + if [ "${sarra_py_version:0:1}" == "3" ]; then + countthem "`grep -a 'filter urlToMatch:' "$LOGDIR"/${LGPFX}sarra_get_from-watch_f02_*.log | wc -l`" + totsarx="${tot}" + countthem "`grep -a 'log after_post posted' "$LOGDIR"/${LGPFX}sarra_get_from-watch_f02_*.log | wc -l`" + totsarp="${tot}" + countthem "`grep -aE 'after_work downloaded ok' $LOGDIR/${LGPFX}sarra_get_from-watch_f02_*.log | wc -l`" + totsard="${tot}" + fi + + + # We want to exclude SACN[34] and KWAL because these are also rejected at the sarra component - Andre + staticdircount="`find ${SAMPLEDATA}/20200105/WXO-DD/bulletins/alphanumeric/20200105 -type d | grep -v SACN[34] | grep -v KWAL | wc -l`" + staticfilecount="`find ${SAMPLEDATA}/20200105/WXO-DD/bulletins/alphanumeric/20200105 -type f | grep -v SACN[34] | grep -v KWAL | wc -l`" + + if [[ "${sarra_py_version}" > "3.00.25" ]]; then + staticfilecount=$((${staticfilecount}+${staticdircount})) + echo "FIXME: yes dir events" + else + echo "FIXME: No dir events" + fi + + rejectfilecount="`find ${SAMPLEDATA} -type f | grep reject | wc -l`" + + if [ "${sarra_py_version:0:1}" == "3" ]; then + countthem "`grep -aE 'after_work written from message ok' $LOGDIR/${LGPFX}subscribe_bulletin_subscribe_f05_*.log | wc -l`" + fi + totsub="${tot}" + + # flags when two lines include *msg_log received* (with no other message between them) indicating no user will know what happenned. + awk 'BEGIN { lr=0; }; /msg_log received/ { lr++; print lr, FILENAME, $0 ; next; }; { lr=0; } ' $LOGDIR/${LGPFX}subscribe_*_f??_??.log | grep -v '^1 ' >$missedreport + missed_dispositions="`wc -l <$missedreport`" +} + +messages_unacked="`rabbitmqadmin -H localhost -u bunnymaster -p ${adminpw} list queues name messages_unacknowledged | awk ' BEGIN {t=0;} (NR > 1) && /_f[0-9][0-9]/ { t+=$4; }; END { print t; };'`" +messages_ready="`rabbitmqadmin -H localhost -u bunnymaster -p ${adminpw} list queues name messages_ready | awk ' BEGIN {t=0;} (NR > 1) && /_f[0-9][0-9]/ { t+=$4; }; END { print t; };'`" diff --git a/am_flow/flow_limit.sh b/am_flow/flow_limit.sh new file mode 100755 index 0000000..9321831 --- /dev/null +++ b/am_flow/flow_limit.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +. ./flow_include.sh + +countall + +#optional... look for posting processes to still be running? + +function wait_retry { + +stalled=0 +stalled_value=-1 +retry_msgcnt="`cat "$CACHEDIR"/*/*_f[0-9][0-9]/*retry* 2>/dev/null | sort -u | wc -l`" +while [ $retry_msgcnt -gt 0 ]; do + printf "${flow_test_name} Still %4s messages to retry, waiting...\n" "$retry_msgcnt" + sleep 10 + retry_msgcnt="`cat "$CACHEDIR"/*/*_f[0-9][0-9]/*retry* 2> /dev/null | sort -u | wc -l`" + + if [ "${stalled_value}" == "${retry_msgcnt}" ]; then + stalled=$((stalled+1)); + if [ "${stalled}" == 5 ]; then + printf "\n Warning some retries stalled, skipping..., might want to check the logs\n\n" + retry_msgcnt=0 + fi + else + stalled_value=$retry_msgcnt + stalled=0 + fi + +done + +} + +#queued_msgcnt="`rabbitmqadmin -H localhost -u bunnymaster -p ${adminpw} -f tsv list queues | awk ' BEGIN {t=0;} (NR > 1) && /_f[0-9][0-9]/ { t+=$(23); }; END { print t; };'`" +queued_msgcnt="`rabbitmqadmin -H localhost -u bunnymaster -p ${adminpw} -f tsv list queues | awk ' BEGIN {t=0;} (NR > 1) && /_f[0-9][0-9]/ { t+=$2; }; END { print t; };'`" +while [ $queued_msgcnt -gt 0 ]; do + queues_with_msgs="`rabbitmqadmin -H localhost -u bunnymaster -p ${adminpw} -f tsv list queues | awk ' BEGIN {t=0;} (NR > 1) && /_f[0-9][0-9]/ && ( $2 > 0 ) { print $1; };' | sed ':a;N;$!ba;s/\\n/, /g' `" + printf "${flow_test_name} Still %4s messages (in queues: %s) flowing, waiting...\n" "$queued_msgcnt" "$queues_with_msgs" + sleep 10 + queued_msgcnt="`rabbitmqadmin -H localhost -u bunnymaster -p ${adminpw} -f tsv list queues | awk ' BEGIN {t=0;} (NR > 1) && /_f[0-9][0-9]/ { t+=$2; }; END { print t; };'`" + + wait_retry +done + +sleepytime=60 +printf ¨\nwaiting $sleepytime for things to settle out\n¨ +sleep $sleepytime + + +printf "\n\nflow test ${flow_test_name} stopped. \n\n" + diff --git a/am_flow/flow_lists/exchanges_expected.txt b/am_flow/flow_lists/exchanges_expected.txt new file mode 100644 index 0000000..9a7fc36 --- /dev/null +++ b/am_flow/flow_lists/exchanges_expected.txt @@ -0,0 +1,3 @@ +xs_tsource_am topic +xs_tfeed_am topic +xpublic topic diff --git a/am_flow/flow_lists/v3_exchanges_expected.txt b/am_flow/flow_lists/v3_exchanges_expected.txt new file mode 100644 index 0000000..9a7fc36 --- /dev/null +++ b/am_flow/flow_lists/v3_exchanges_expected.txt @@ -0,0 +1,3 @@ +xs_tsource_am topic +xs_tfeed_am topic +xpublic topic diff --git a/am_flow/flow_save.sh b/am_flow/flow_save.sh new file mode 100644 index 0000000..a7fbbb4 --- /dev/null +++ b/am_flow/flow_save.sh @@ -0,0 +1,3 @@ + +mkdir flow_save +rsync -a ~/.config/sarra/* flow_save diff --git a/am_flow/flow_setup.sh b/am_flow/flow_setup.sh new file mode 100755 index 0000000..0316cc3 --- /dev/null +++ b/am_flow/flow_setup.sh @@ -0,0 +1,172 @@ +#!/bin/bash + +# make sure libsrshim is off + +# if running local sarra/sarrac versions, specify path to them with +# $SARRA_LIB and $SARRAC_LIB, and $SR_POST_CONFIG +# contains the path to shim_f63.conf (usually in $CONFDIR/cpost/ +# shimpost.conf, but specify if otherwise) +# defaults: +#export SR_POST_CONFIG="$CONFDIR/post/shim_f63.conf" +#export SARRA_LIB="" +#export SARRAC_LIB="" + + +# argument could be: config, declare or nothing. +# if nothing, do the whole thing. + +#export PYTHONPATH="`pwd`/../" +. ../flow_utils.sh + +echo "FIXME sarra_py_version=${sarra_py_version}" +export sarra_py_version=${sarra_py_version} + +testdocroot="$HOME/sarra_devdocroot" +testhost=localhost +sftpuser=`whoami` +flowsetuplog="$LOGDIR/flowsetup_f00.log" + + +if [ -d $LOGDIR ]; then + logs2remove=$(find "$LOGDIR" -iname "*.txt" -o -iname "*f[0-9][0-9]*.log") + if [ ! -z "$logs2remove" ]; then + echo "Cleaning previous flow test logs..." + rm $logs2remove + fi +fi + +if [ ! -d "$testdocroot" ]; then + mkdir $testdocroot + cp -r testree/* $testdocroot + mkdir $testdocroot/bulletins_to_download + mkdir $testdocroot/bulletins_to_post +fi + +lo="`netstat -an | grep '127.0.0.1:8001'|wc -l`" +while [ ${lo} -gt 0 ]; do + echo "Waiting for $lo leftover sockets to clean themselves up from last run." + sleep 10 + lo="`netstat -an | grep '127.0.0.1:8001'|wc -l`" + sleep 5 +done + +mkdir -p "$CONFDIR" 2> /dev/null +mkdir -p "$HOME/.config/sarra" 2> /dev/null + +#flow_configs="`cd ${SR_CONFIG_EXAMPLES}; ls */*f[0-9][0-9].conf; ls */*f[0-9][0-9].inc`" +flow_configs="`cd ${SR_TEST_CONFIGS}; ls */*.inc; ls */*.conf;`" + +if [ "${sarra_py_version:0:1}" == "3" ]; then + cd ${SR_TEST_CONFIGS} ; cp -r * ${HOME}/.config/sr3 + cd .. +fi + +######## No need to convert the configs. They are already in v3. ######## +# echo "Adding AM flow test configurations..." +if [ "$1" != "skipconfig" ]; then +# cd ${SR_TEST_CONFIGS} ; cp -r * ${HOME}/.config/sarra +# cd .. +# if [ "${sarra_py_version:0:1}" == "3" ]; then +# if [ "${sarra_py_version:5:2}" -ge "54" ]; then +# sr3 convert ${flow_configs} +# else +# for i in ${flow_configs}; do +# sr3 convert $i +# done +# fi +# fi + + if [ "$1" == "config" ]; then + exit 0 + fi +fi + +passed_checks=0 +count_of_checks=0 + +# ensure users have exchanges: + +echo "Initializing with sr_audit... takes a minute or two" +if [ "${sarra_py_version:0:1}" == "3" ]; then + sr3 --users declare +else + if [ ! "$SARRA_LIB" ]; then + sr_audit -debug -users foreground >>$flowsetuplog 2>&1 + else + "$SARRA_LIB"/sr_audit.py -debug -users foreground >>$flowsetuplog 2>&1 + fi +fi + +# Check queues and exchanges +qchk 3 "queues existing after 1st audit" +xchk "exchanges for flow test created" + +if [ "$1" = "declare" ]; then + exit 0 +fi + +testrundir="`pwd`" + +echo "Starting trivial upstream http server on: ${SAMPLEDATA}, saving pid in .upstreamhttpserverpid" +cd ${SAMPLEDATA} +$testrundir/../trivialserver.py 8090 >>$trivialupstreamhttplog 2>&1 & +upstreamhttpserverpid=$! + +echo "Starting trivial downstream http server on: $testdocroot, saving pid in .httpserverpid" +cd $testdocroot +$testrundir/../trivialserver.py >>$trivialhttplog 2>&1 & +httpserverpid=$! + + +echo "Starting trivial ftp server on: $testdocroot, saving pid in .ftpserverpid" + +nbr_test=0 +nbr_fail=0 + +if [ "$1" = "ready" ]; then + exit 0 +fi + +cd $testrundir + +echo ${upstreamhttpserverpid} >.upstreamhttpserverpid +echo $httpserverpid >.httpserverpid +echo $testdocroot >.httpdocroot +echo $flowpostpid >.flowpostpid + +if [ ${#} -ge 1 ]; then +export MAX_MESSAGES=${1} +echo $MAX_MESSAGES +fi + +echo "sr starting " +if [ "${sarra_py_version:0:1}" == "3" ]; then + sr3 start + ret=$? +else + sr start + ret=$? +fi + +count_of_checks=$((${count_of_checks}+1)) +if [ $ret -ne 0 ]; then + echo "FAILED: sr start returned error status" +else + echo "OK: sr start was successful" + passed_checks=$((${passed_checks}+1)) +fi + +abit=20 +echo "waiting a bit $abit to let processing get started..." +sleep $abit + +# For the watch. To notice the new files after it's been started. +cp -r ${SAMPLEDATA}/20200105/WXO-DD/bulletins/alphanumeric/* $testdocroot/bulletins_to_post/bulletins/ + + +if [ $passed_checks = $count_of_checks ]; then + echo "Overall ${flow_test_name}: PASSED $passed_checks/$count_of_checks checks passed!" +else + echo "Overall ${flow_test_name}: FAILED $passed_checks/$count_of_checks passed." +fi + diff --git a/am_flow/offset_flow.sh b/am_flow/offset_flow.sh new file mode 100644 index 0000000..ab4d1ce --- /dev/null +++ b/am_flow/offset_flow.sh @@ -0,0 +1,23 @@ + +#must be sourced... as in: +# . offset_flow.sh + + +export SR_DEV_APPNAME='sr-staticflow' + +if [ ! -d ~/.config/${SR_DEV_APPNAME} ]; then + echo "mkdir ~/.config/${SR_DEV_APPNAME}" + mkdir ~/.config/${SR_DEV_APPNAME} +fi + +cp ~/.config/sarra/credentials.conf ~/.config/sarra/default.conf ~/.config/sarra/admin.conf ~/.config/${SR_DEV_APPNAME} + +if [ ! -d ~/.cache/${SR_DEV_APPNAME} ]; then + echo "mkdir ~/.cache/${SR_DEV_APPNAME}" + mkdir ~/.cache/${SR_DEV_APPNAME} +fi + +if [ ! -d ~/.cache/${SR_DEV_APPNAME}/log ]; then + echo "mkdir ~/.cache/${SR_DEV_APPNAME}/log" + mkdir ~/.cache/${SR_DEV_APPNAME}/log +fi