From 5a58f48352ed2830d5c9d6649784d75fc0eef6e1 Mon Sep 17 00:00:00 2001 From: Illia Antypenko Date: Thu, 9 Jun 2022 14:04:51 +0200 Subject: [PATCH] Fix encoding issue for non-English websites, closes #454 #466 (#482) --- .gitignore | 1 + lib/config/defaults.js | 3 +- lib/plugins/save-resource-to-fs-plugin.js | 3 +- lib/request.js | 23 ++++-- lib/scraper.js | 4 +- test/functional/base/base.test.js | 8 +- .../binary-resources/images.test.js | 69 ++++++++++++++++++ .../binary-resources/mocks/index.html | 11 +++ .../binary-resources/mocks/test-image.jpg | Bin 0 -> 15801 bytes .../binary-resources/mocks/test-image.png | Bin 0 -> 12192 bytes test/functional/callbacks/callbacks.test.js | 1 - .../circular-dependencies.test.js | 8 +- .../css-handling/css-handling.test.js | 8 +- test/functional/encoding/hieroglyphs.test.js | 41 +++++++++++ test/functional/encoding/mocks/index.html | 12 +++ .../html-entities/html-entities.test.js | 4 +- test/functional/redirect/redirect.test.js | 12 +-- test/unit/scraper-init-test.js | 4 +- test/unit/scraper-test.js | 10 +-- 19 files changed, 183 insertions(+), 39 deletions(-) create mode 100644 test/functional/binary-resources/images.test.js create mode 100644 test/functional/binary-resources/mocks/index.html create mode 100644 test/functional/binary-resources/mocks/test-image.jpg create mode 100644 test/functional/binary-resources/mocks/test-image.png create mode 100644 test/functional/encoding/hieroglyphs.test.js create mode 100644 test/functional/encoding/mocks/index.html diff --git a/.gitignore b/.gitignore index f0567b90..f010ea32 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ package-lock.json npm-debug.log coverage test/e2e/results +.nyc-output diff --git a/lib/config/defaults.js b/lib/config/defaults.js index 1e79fe63..8780e83e 100644 --- a/lib/config/defaults.js +++ b/lib/config/defaults.js @@ -48,8 +48,7 @@ const config = { ], request: { throwHttpErrors: false, - encoding: 'binary', - //cookieJar: true, + responseType: 'buffer', decompress: true, headers: { 'user-agent': defaultRequestUserAgent diff --git a/lib/plugins/save-resource-to-fs-plugin.js b/lib/plugins/save-resource-to-fs-plugin.js index 0e7be17c..dfb5e3ba 100644 --- a/lib/plugins/save-resource-to-fs-plugin.js +++ b/lib/plugins/save-resource-to-fs-plugin.js @@ -20,7 +20,8 @@ class SaveResourceToFileSystemPlugin { registerAction('saveResource', async ({resource}) => { const filename = path.join(absoluteDirectoryPath, resource.getFilename()); const text = resource.getText(); - await fs.outputFile(filename, text, { encoding: 'binary' }); + const encoding = typeof text === 'string' ? 'utf-8' : 'binary'; + await fs.outputFile(filename, text, { encoding }); loadedResources.push(resource); }); diff --git a/lib/request.js b/lib/request.js index 4ea4e76b..8d993093 100644 --- a/lib/request.js +++ b/lib/request.js @@ -1,18 +1,24 @@ import got from 'got'; import logger from './logger.js'; -import { extend, isPlainObject } from './utils/index.js'; +import types from './config/resource-types.js'; +import { extend, isPlainObject, getTypeByMime } from './utils/index.js'; + +const TEXT_RESOURCE_TYPES = [types.html, types.css]; function getMimeType (contentType) { return contentType ? contentType.split(';')[0] : null; } -function defaultResponseHandler ({response}) { - return Promise.resolve(response.body); +function defaultResponseHandler ({response, type}) { + if (TEXT_RESOURCE_TYPES.includes(type)) { + return response.body.toString(); + } + return response.body; } function transformResult (result) { switch (true) { - case typeof result === 'string': + case typeof result === 'string' || Buffer.isBuffer(result): return { body: result, metadata: null @@ -41,14 +47,19 @@ async function getRequest ({url, referer, options = {}, afterResponse = defaultR const response = await got(requestOptions); logger.debug(`[request] received response for ${response.url}, statusCode ${response.statusCode}`); - const responseHandlerResult = transformResult(await afterResponse({response})); + + const mimeType = getMimeType(response.headers['content-type']); + const resourceType = getTypeByMime(mimeType); + + const responseHandlerResult = transformResult(await afterResponse({ response, type: resourceType })); if (!responseHandlerResult) { return null; } return { url: response.url, - mimeType: getMimeType(response.headers['content-type']), + type: resourceType, + mimeType, body: responseHandlerResult.body, metadata: responseHandlerResult.metadata }; diff --git a/lib/scraper.js b/lib/scraper.js index 040a9cd9..339aa301 100644 --- a/lib/scraper.js +++ b/lib/scraper.js @@ -13,7 +13,7 @@ import { } from './plugins/index.js'; import * as utils from './utils/index.js'; -const { extend, union, urlsEqual, getTypeByMime, getTypeByFilename, series } = utils; +const { extend, union, urlsEqual, getTypeByFilename, series } = utils; import NormalizedUrlMap from './utils/normalized-url-map.js'; const actionNames = [ @@ -170,7 +170,7 @@ class Scraper { self.requestedResourcePromises.set(responseData.url, requestPromise); } - resource.setType(getTypeByMime(responseData.mimeType)); + resource.setType(responseData.type); const { filename } = await self.runActions('generateFilename', { resource, responseData }); resource.setFilename(filename); diff --git a/test/functional/base/base.test.js b/test/functional/base/base.test.js index 383bbda8..8ac3dc3f 100644 --- a/test/functional/base/base.test.js +++ b/test/functional/base/base.test.js @@ -51,15 +51,15 @@ describe('Functional: base', function() { nock('http://blog.example.com/').get('/').replyWithFile(200, mockDirname + '/blog.html', {'content-type': 'text/html'}); // mock sources for index.html - nock('http://example.com/').get('/index.css').replyWithFile(200, mockDirname + '/index.css'); + nock('http://example.com/').get('/index.css').replyWithFile(200, mockDirname + '/index.css', {'content-type': 'text/css'}); nock('http://example.com/').get('/background.png').reply(200, 'OK'); nock('http://example.com/').get('/cat.jpg').reply(200, 'OK'); nock('http://example.com/').get('/script.min.js').reply(200, 'OK'); // mock sources for index.css - nock('http://example.com/').get('/files/index-import-1.css').reply(200, 'OK'); - nock('http://example.com/').get('/files/index-import-2.css').replyWithFile(200, mockDirname + '/index-import-2.css'); - nock('http://example.com/').get('/files/index-import-3.css').reply(200, 'OK'); + nock('http://example.com/').get('/files/index-import-1.css').reply(200, 'OK', {'content-type': 'text/css'}); + nock('http://example.com/').get('/files/index-import-2.css').replyWithFile(200, mockDirname + '/index-import-2.css', {'content-type': 'text/css'}); + nock('http://example.com/').get('/files/index-import-3.css').reply(200, 'OK', {'content-type': 'text/css'}); nock('http://example.com/').get('/files/index-image-1.png').reply(200, 'OK'); nock('http://example.com/').get('/files/index-image-2.png').reply(200, 'OK'); diff --git a/test/functional/binary-resources/images.test.js b/test/functional/binary-resources/images.test.js new file mode 100644 index 00000000..05d07916 --- /dev/null +++ b/test/functional/binary-resources/images.test.js @@ -0,0 +1,69 @@ +import should from 'should'; +import '../../utils/assertions.js'; +import nock from 'nock'; +import fs from 'fs-extra'; +import cheerio from 'cheerio'; +import scrape from 'website-scraper'; + +const testDirname = './test/functional/binary-resources/.tmp'; +const mockDirname = './test/functional/binary-resources/mocks'; + +describe('Functional: images', () => { + const options = { + urls: [ 'http://example.com/' ], + directory: testDirname, + subdirectories: [ + { directory: 'img', extensions: ['.jpg', '.png'] } + ], + sources: [ + { selector: 'img', attr: 'src' } + ], + ignoreErrors: false + }; + + beforeEach(() => { + nock.cleanAll(); + nock.disableNetConnect(); + }); + + afterEach(() => { + nock.cleanAll(); + nock.enableNetConnect(); + fs.removeSync(testDirname); + }); + + beforeEach(() => { + // mock base urls + nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/index.html', {'content-type': 'text/html'}); + + // mock sources for index.html + nock('http://example.com/').get('/test-image.png').replyWithFile(200, mockDirname + '/test-image.png', {'content-type': 'image/png'}); + nock('http://example.com/').get('/test-image.jpg').replyWithFile(200, mockDirname + '/test-image.jpg', {'content-type': 'image/jpeg'}); + }); + + it('should load images and save content correctly', async () => { + await scrape(options); + + // should create directory and subdirectories + fs.existsSync(testDirname).should.be.eql(true); + fs.existsSync(testDirname + '/img').should.be.eql(true); + + // should contain all sources found in index.html + fs.existsSync(testDirname + '/img/test-image.png').should.be.eql(true); + fs.existsSync(testDirname + '/img/test-image.jpg').should.be.eql(true); + + // all sources in index.html should be replaced with local paths + let $ = cheerio.load(fs.readFileSync(testDirname + '/index.html').toString()); + $('img.png').attr('src').should.be.eql('img/test-image.png'); + $('img.jpg').attr('src').should.be.eql('img/test-image.jpg'); + + // content of downloaded images should equal original images + const originalPng = fs.readFileSync(mockDirname + '/test-image.png'); + const originalJpg = fs.readFileSync(mockDirname + '/test-image.jpg'); + const resultPng = fs.readFileSync(testDirname + '/img/test-image.png'); + const resultJpg = fs.readFileSync(testDirname + '/img/test-image.jpg'); + + should(resultPng).be.eql(originalPng); + should(resultJpg).be.eql(originalJpg); + }); +}); diff --git a/test/functional/binary-resources/mocks/index.html b/test/functional/binary-resources/mocks/index.html new file mode 100644 index 00000000..20f1cf2a --- /dev/null +++ b/test/functional/binary-resources/mocks/index.html @@ -0,0 +1,11 @@ + + + + + Index + + + + + + \ No newline at end of file diff --git a/test/functional/binary-resources/mocks/test-image.jpg b/test/functional/binary-resources/mocks/test-image.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2fc54b460fa0f7aeb06b8d9a04d481a9f36615b2 GIT binary patch literal 15801 zcmbWec|6qL`~N?pw1|l8rb5V;eK(bCNkWW$%90SW4~8j1mIxtD>l}_8PJ_F8PM4x2$>phm~({+7RQ1D@!P(K8%B!Fcl2 z=`+9!D$fB`r>8&0K!5!B(bv$00QW%*tjF0d$=o<`-uNNoWiNKw=ZPOr3f?Sj1yNBD|}sMu9`1w|!g6;-WU+B&*=`nOH*o0&r`EUg?IogO(qc5(Ib z_45ya2L^?|h=`1O86A_9{3aze?QQzI?3_=zdHJ6UzLb?$R902j)YkoIYHn$5``O;n z*FP{gG(0joHZwanzp%Kp{AY!@xwXAR+9mJpAK3-8=Ralv&;MBV|FVk}u(3C0tD3**1#x;t)z%9=nj)9oy=>TV<_rAe4T)37dGq4U3d+myTgr=Q9-<5E?H>(zhb_E5W$c{;RDcF`b2lIEO|>cy{a3v*ERZeJ_vr?PHmvO||V+rV+} zhNCMfsI$vZ7SG%8mY12Oy{Np~l@+dGxokQB?Z}a_n zF7#yF?73!o{_vdSx9*nrt9~wm+L!qXKVJP#Vf~559DD>HR3vo8 zIP89Ud5(F*+(cKSap|>TfP=T?ulzMZDaY!*sY3OJ@@TQyFJI)M#Dm>@4?zqe5N(b< z3UO&pp!zF3a`lG*ICHInBJ$W#CbLLBw#hMJhVhx&&c8J@O;QbTE=YfLpK$+MuAyhP zEe_k9Snh7lu>S}Ji+gRP@wL=U z9H8FtnVw?)tx*|~%$4`J-Ta;Q%yYraFYq?4xc9_(rVGsQckf^K*K62x$^ibi=IYif zg{MsGbGyfUN8RaPB_?q%Ri}GMPUIxsHG4B1zH&BXoT5s#9aJ~zgUbQ6TmO*IRh^m8 zwcNnr@H=i^;FGh4%0}ya`A5TnxZ0#CgV3*=-jl-lF<(FT@D;5U4bOiKvDM}vIJX^w z;(`X3UP>X(&(ud&m6u-;o4jn6`#NMn$fwe%Gk!_sXG_Wm1JacAE6sW9a;fbY4jBS(j4Nky}kQ;l|_Z-3k9awm-i zEQ~^V6BE2u!vb4<#=bE<(pP1l(CbW>+^$~12};ho!ryiLb-k%=-JuF&ZFp2a>hKqh zCQfXE_DgNsnlI^LZOg|xh{}f`7v{5}ZNtwI=H#^}rWLX^Z;SVoQR)YlNZ7Yn%# zS}Jn{t0^Y3YvA}kmM5<#l4V6??E%Sv)n6Ebl%{17C7E8zfp`D>~9H_RDrjDjB?ju*q* z(vs&sMimH~y7+BtOkV0(vHzppOU?;V%RdA$g}M#*L&{py1O4VGukK#Fb)hO=$Yb>N zvqJJT>PH10;yA_mZN$*Xq&$Xcljxa9r2R1ilXpUp(-6AH5EglbUU_--%)Sn41A4`1 zlm59~vD3R>zD{>D>Vtv=m<0-yJcuQMke^f zyZBsZc5VuBBcP3vdmlMBxU=+36Z{y=>n?=G(R`fn1rt4F+`RUR8YUjK)$Y{a87ElaPc5T0iCoE05bu>F?W5>VY zy7u=)NcMdj(xJjdRM5+>YYTx!p6{Q~Ykqp4kn5|6D)WrX!V8mfo%pyH=A&FM)!R`$*45_Q3XG@X}&mMMy=wJ8D@XY zuq)zU2-T%t@7s{~RjejZ6DdU3@LR01zxJDkXKwQeHEZMV6WqGo3A!#j~!2uB2)n(~XltIHcGV1I$ zXwUqfflz)G_iZ|I>Vf#@s)J;N3YoZdCNQ2P!f$Hn1QLwH*YENqOkx!HO{4sZ3wpi;!_H@7aKj+nUYlep2> z@Fbs?Ndvm3hyO-jO&HQ(juAz_hTMc4_WnfCB?C^#@VB}wsW)3UC@{#lA9)dEADL}@;s<=j%>%{sU9vrS}WIj5U8nvhZyJe__YoY2os9Fu=wsYzfII;!F0@^20O z#pYW^>C@@VW^?vmSxUJsW3OYKaj^k3<%*0ZJ3*w}y|NpZ4SscAc&ZP~-TI)cJ^oF% zdj90QFEsuS_UII;sx~6%_Ei|eP)L4Fxz%?)Sf3dPuyht@K3Wuc{I>AUYLjEUfB0+A z^p46LpRxoyQEommbM()m-R|p9^== zO7T6aB;W89vV6FuTer2Od&ZLun}K{Brnji!^zy+!cr;C63ofmdUp6Hrrpa3W@!a}9 z;X?&jKsnnI;`S#C&=YV-Qvi$m!$+;$R&tejiSQ?hJ-^@}zt+3(Hs&+O)eZ0JAx2mJ zw<`LFY?j|#?-dN|9H9TXKafJkxd#R~KdnynZZN&nQ72>0IKbK+vPMxy*bX$+Q(DH1PWSfv&=C0-i_=zEAWq zm|X2$PTtjDd-(n+g(oMr87qToA3Z4fy=#qbc>=ebG?j48?O5yB z+4jeFThLUI9-d?sB~mGQbT$^pOlwa9M`BoFbo4|l<}xhm#+ngmeKH8bugPcKhfE2X#S>oF!)@_phWRy`!hPdxSG#T9DJX|Ln0zVZ}!^w zm8N{DT2ew!>+%L%WlngzatU3JqYY3!3yU z5LPKX?-%%aeLz%kRYdX97ZY z*z(7u9i+;SccNa(t;ig^R9S(PsdxH$8k-lr4Ce;$?mtjrJFfPQ8cMWF z56FRM-0MR({w1T}*KMe_&*%{zf@duFSOxMWK%MRu-6Z}yqNZ1rgg*FAs?Pf|srli! zHoXzP*LH-yJlW4W1f5$*k@2-(rW*g5Tv}{Fih&7c8)W6-StEQdG5r3)d+Ygj;XSvKZpYDweaYu7+_wU{?!)T4UHmD^3F@7L&(s@V zEN&=8RYYIzyD3-S`DHTE)A9LjDcv!ln^eN{gzIk92nBbj$*`Y6rjKRjI$BydaOs`7 z#HPU|iLT~N12%n;0<@N5_ob_$9WT%!2WkRFEc1Ss%!XTT6x*hukcs=9*~rEq>~zUK z%PZXkcgSPu3!QvIGeDTaw|~N?8h;i>;-=MYvUC$haioy9rUv)ZiEvsCq5$J+E}7? z&$@qw!;Hi4Rrv{f+dr^mmf5UZ;0iF&4I)0}YJa#QCYeQx>RYURBJ;*Zm?(*IKJY-I zcS2YxGA`9=sPh*{OfQQh0Ze;xZH#U|8TF$E`3%eiBgK%lX29_n#zFii!f@PnhNNq& zuXBLnaIZvr%GinNuk=zgkqrLmUY01>*=?4{F`2&Yr;bS`Jb_aiKb!viVs5IgZAX9n zFED&XL>eaAn_p3I(@m8L7%tQWh$kRnd%PRK&?;2u$rPytp=uNE=(C5QV+`O{tc_a} zO`LkJ7b7gY^4fsJe_LVX%9&cnBXY~PR}r8>6W%yHjQV_#heJCo*Mv(H#N~jUh}vtv zsW|ip$fH+^o4d1doT<}${!hFAA-BDf;`nir5wU;b<@yHRAxgcjA`bBPaFG?=AxtgV zlhA*I?2=QJ@oCXA?^1rvQ~IAx`MtUi$2C8_Bl++RMU}o0>sTuUt0d1`0NC>{xs{S9 z4nfDq0Vl)34QQuMQ8Zk}s>1UdXQiXF6q4LL&$`HAE^0rcS6KLE&N8o6d;jW|P+Z|_ zCD}vJJ4nJIXu9?gM7Tfo5_K-0oLE4$9C+ffSeLYH^@phC=J@K8i0A7>yd(E_X5N9M zKs5Xi6rKoi$CBMqoviVO8{g#w3dS?V`xDmWD zCLGQzVTM8AVDkvoR8peRpbf1DJDrRoctM-NFOlrq(xcTRyqiHD@$9d%{swC%m&XpZ zX)o;?B84rg^vJKAv~6>rdsX`+tiwp|{hrdH;+7NRVGiXnE>0R@W$a+`*6Bmgd1V3k zlD=fDM9lY(c3ju~#QD4*rj&%XXMxfh!E_B`QuVC1p@$%y{>4GtJ^AG!hm1hw) zu10!Hz^)}w#1PjH($wvLsIl-pP75+Mc-6}7q_uFHZiD@!O>j|&0xS#3tEsSxxj5{B zP?{M}tL&Gw&AxKkEYC^VUT?`Gi1~%LTlw*@tSgAuy^^C#j{-9#0MW+O9MHUoM&x`# ze=F>S<`sC<3}dskV*kz|=!`4W>Td3n-){kedhB#(cWgx`c$g8^i>C6GQZ7q>$Bx@g z&22eoJ@qG@E*@sm)rx)prStra-omRU%f1hY#?8hNkMYA) z)jfeILc(l74CXY#oPcXJqBkUkxcvw!$|`>-4H52+>y+1xhx>BS77jsWu(k8khoG@3 z(j$WOn)I{?OD=p|yJGMmeKPOeXYJgFAXgP7!QcWk;T3wD0dN;!`twBc!*@vvL=n7x z;VN4pLU~~95M+401x67!7p8s~ZMP^+eRS~wi!%LK;i*W!l`r~Z^A5z=GdIoVGYm=( z_Kjh$QdLuP@@&tnDF#X|*(K?;ScD3`JD@*6VI-&En;l3Ncy#!Rq8`6fD^sa&J9yIc znGqr(Xj=vVN3F4ch{rsT+nKcw0C3s=0Jt8YtHDEiu@(F+Z}xLjvOXT9eOxXOpR|b_ zt+{?GB|Yl00*IjcJUI(A+xTE98YSerP7%m*dfVUFcLd?L(#(-y6d?$`)b^nSayd}TTzH%e*PAchIU3|pNZZUIwZ^-mhsom#9e2?Qg{dxL3Y@Ff- z0BYs^t;=tBQZGFVKvsN;4-Uuc^aowpJdU%a4h_kXU=&B{yxs6ae67bpq(W{z~OIuRw^u~ z`G-uwGNuSn>2Vz#i4K_10ugr(a))M>{X(Q~C-rMt%HFE0+*p@nryFKd_Mvqok2Oe* z(e``rurWg+)9@O-IJ3r`pGykVFl6I6jBpRe0ZdoAWCsdEELKH6U6|-!Mc&Iz94zwP zvA-r8Q52RZ=y%@<7*_`;lPoWS8vw8zMXk@ z&H&5nT$MQcTA(Y6aR)uhvo|$6{GDHI_V^6(M6Bw_${o<_k}i|!2N&8d)jK-JPppo% zm;P$%F)pYmLg!Oew%%)}nzi)3_`E7Qb@$IWHDQ65c4nL+Y(z0fHma-Np5(7`L$Sh7 z-zVz4{d{8SLQK1#K+yD9!&i1cU!XDM0i5t4AA%f_Tr0FkX|rcyIm*(lRBBxU1esW? zANT0mYaD2f1sgns;!>Pc7@pqXnD71ar$<4sEx^}7+(Z4pmnPHE#R8Pc23?nH*N)aJ5e@&92|d&D!zrLojO1JEr53VJmmyJ z_2AQ`!6%sB&la)#kHfkptTLlFJ#O6bbB0|z)^v4x>D|FJIhe3@lkAm^bbS21zV@vA z+U~-`-}`hU{`>Q&0)S+s!XuOdgm=MblN~MI&gPg@1Uhi;j+agj6TJ>WjJ%=!yV$ZQ zyyWZ#{g{mgHX3sYrWCPu`n__bJuXnBxZeXU5aI;0qV=v)KSQS1HpY4H9ArbL^Oa!q zh>Q1@r%qOoD*``7m!ulpxt^;FKC3c&zFrEV zJw|$J`@lplok%S=*0$^0{$^l3u(oZ>b4m(;R=tan!qP%XdN+YcMpBm^h*`h*e7|5Y z?+89;U_N4K%o(2Zrr;!w%VdT3<5hkGx}QvP17^Ttabt=siK4Rd>X88uV>m&+gJrI$ zSt3T^_X`W)k+U49cdxCwqQ*A)j=?%>E%?1dEgTeRi2HOJMT0FVT>+^cy#{9(bxkA9 zO<&*GIe{Gxr=2vS7#LApXk92~3`Bl0#DKz4q_C)gtoYhC>Y+r}-<5(GWF&L(0LpO; zZ527)F*W{ZxCzOjDMVtK!Zft^O0LiLv@6a0+UyI zlAnqWLPbu<(erHO8ZXZPv+Z{Z9+_<$szavSZRkUI7P{N}|^B@dH(s*LpSL_vrC5!=Ek$`gxl`_5#c#J5Z zsHO9y=041E8Ro$h%36<&wQdb0)bA6^kLh}NgD@wJD%e#q*~(C8q4Q) zL2GjO{q$|}g34oGazdILXVp(Ly4`m4_H|$adJO;uio^@dwq9;k$*{32)kLXjxanQ% zz>J>FV&klJv*Y*I^f+E;bzj~e-U}bT8xS91XXTsgTj>mDT1dIJ;^(H*7bH$v-K9OZ z%OJpJpwToHsx1N6h(4j?G>9-Q82jV@LF|dX-{n@W=Tz~`u3(U1U<%S2OK^cw&*$xu zat^*ApcFIsbDk+(vSt1rk8M-lGRkUTFz0j#q8_FYuVU`&N)!{VZ802$!L;W2coG0WJ!~O&xC#C+IdBOIX1oeCyq1e~XUSy8pW5%}7)^{&rXB9Yq(Q z#|>#sfF7;G94ca5gatpCM4UiQ-x1FVC*3DM+qLJ7trv<_|H(g|@UX;Z4>}BE!f-Al znJV(j`=hfK5!~y158uZ>WA0F6S%N_mx(F*T3i<-vt+l4h-Y0WP+;@Y<#>Ccp7EWzx zNc~5L9Q><8YGM=-PlqeN`Oh`kU8oHfmnoV+?xT1$Bi|*=Q>ZbatVSElX6EIB5Og^0d}QqNx@|l1iMg)je&a<#XZm z?{UnvIlchC_>&jocY2xR^KY9ukOE)@?WWwP{bp_ksfalDR@Uu^)5cI0OD`Y}(xVW7 z#VWz^j`#bi962^CYSuKXp2Dh=Q4Qs?IvBh-RasM%sz(fHg+zy5CfCnVa)OJ=`KI-Q zJG_yrk`wqMdGyqVS~6@I2um^Mg#I=rgGoUIFrH;byvop^jdQL!Xr*PWr*W|1KEhimdqc5s+6c4kRPEE!(R6TE;TU7Nsl3s;LHZ zPr_t}t-&rUrPaAe)k@ljDS|T=kIO+Y9=xbTMNeKVvrxAr!tU|>d`}d7Ty=eX>=tt3 zHSXGuWm~R|ONV|$+Lc2| z)~&h)U`zzE-Vd3cKoyXqTw7MfbW@~zIjaS(A9aazj7&6#6bf!Qb|bSPBhAOcWoq@%!<0TDt@8I5%V zoKVph;R-L(BR2(9nydN@z|=F}5I)*B3TXRwr-{fY+C{ROK5_TO`h|_#!xA3`B$l*N zQYRM}5^FT$zMkA4HiUJ-s5~JQ&o;{aLr|l%JUX0q1?z_9pz<~(H>KG>IsxssN#EX+ zIcSOKF53n0QDp~`J5}Abmg_{(Ku8kvo6lsVRs3pT$+KPIdGWfc9ROU}MInz5<#b&1 zsSfeleh;sep*1a*Q;SHt7a(9JNd=X{DtyvJIr$PW+kVHo_foWwjat6A(bylm*>g>Y zpcC><&Q~vW0G7H2Uuh626=Sn*p>+x}FJhDGHZWJIw~4S;STyD$xpJ;BwP6Th@cH*j zK)H%g!>$Q8-~85Kyhh>kV&{2(@p0qpLet)c$kGieZ##uW2`>yRpe5vh=~tnH?o`vc zGf1+5-yta3A-2duqV(4EroiNQdjxjuX7KP{I?x(S$cF%nO@Dw8Op%dO^a`EO8p2@a zxEt+~_%8HLU)1}RrO%cNtk&~B>AFMMp@cbHYmqCzhbKK*)4x0n5R1>>@~_9JmCX=ZJT*-0`B0VLrRLHwKgnrEPZk1GzOL<%A1vW)yJy2xC$YL2@0PHA$&g zzvPH|SN{!G3=9C_>ijE3bY>cqL%0w3Q@(<&)~FcpLina*i$ zb%@CPT-MSRKwM?b`y2Uc$B=09rMDH3GWT#YaAA+JZeO_*M)YRQe`LCi` zhRHlgk&s#>?o57ULtQzeXvte;oA#Lc)M&=|53ohXHlK-YkMT@g~`9m3pq^RnN_AG3rHy=zjJUhC}p7oJhIo^)KgtSlv^ z^Gnty|02CDyA0c>ytFZM6)$By^Ar^Wczt$0l@m-FNh1Mn81|lOM4ou~v|1BfI)G}L z6nH$2aR2m2Vu+v@)i+&e<52o$cs`dMsxlD5ND)PVh$3r76n(fjF!6p-vxS$NSCu-) zIW~E8r!PYAO(z3+M}dLShSw%+>5`EJf9y^X48M#nV|e;5bW93y=}kwNi1u&N|DIEz zDgi?w2SqSE1aTq)$mlunc<8>e`?e9IINZme1{Kg9FL zBH8^TRlfIbT5hs@-|PP0*P8U2#>?WH)Pvu@+j+Y4$M}b#-Bwms?0)TxeMu#eYu}`@ zpnvFKDs6_G6%FfzP%l`MR5D172d^>02v$W5^5xe%V^z`hVG3%;HZCwj7QUp$DpSgM zXnEpGhoD&OG5M*N<-qRA#Z)pik6m1jsSQx<=U(gXM#AS9V=V2iq^R7@^o+dS)I%^U z)XB7gv@!bDqh`+CFfoffxgsV`g3``(Q;w6`R-(-CQSRGuySAnl@?sjchj zqRK;1x%?l7^aqegmv`z&xFB;ndOD-gPKGGmgynXDoKxraC-JmK1fTfm;K$n3HV#ls2|exS?+iZSo)SPA6}#-yRD@$tBOjk3#Vl%B_+bh&sXkWk3j;+crHM zJ|bWYh&)W}gbgaJqL|?33wXRFU-KpqS?tMY3Shpu^o-FiznJk65T3oW2V^&)fZVW> zzqw&$wno@xblW#b^W^2e;vrzKiKz*5QejTI2I;|G$Iv6JYspc0hPiGdwEj|1`RDJc za;fJEG&<6{RXzYdoJh37UFFJ9Z5rfXu36Ss){MIk^&n|`)hT! zBU4(!-3Ky?_yxb+zVIA}sJf|=(C5V9lz?i=xuhCHxx6!hXrG@5kJm&!*Q|K!D>1XIcAGR@C!Atj0k=yKVFIL-^h>r+XJ3BgDb{c zO(45~cKN|r$~#6jx&&2;=_E$pH?NtMRRnfc9K2oMTmrTeNmCzZ78-PQIiVvR>Scek z1_hVLM9~p6HH6nN*tUk?`9g}PZ?ZSew3~HYD2b)vW8Ig6xj42PYc>|@d_ZbJR5wXO zy-nsXMu$c-S&>Uz7pKODDpM~L zq2HzBvP8+kb1&npsv{H(ilpW_#B0|+i8atur9e*VVV}YA&!cPwM%6fDc*x3Z={7<*hli^Jfhw{C~FYo$rWf zoaeg88W)s0?UVUX$0shABYW`eV6$ny=uMBFyFY13kFzm?pFde{fcRLk{J)Wz=MkBG`IpSpkcoQ>u)Coh zsZbyI{`&q!#M>vCjvk`{wmHN`NQ4;>^>xC&tX8t;{Ua%vjg#d*L&SsHsa`N=sw&y< zBUSMifNt{d+l5}JZ`JV5Bbz?(M|l?O@A%y5NhnEC^{x*|I3w6DBYvJ|(0-0<6>W+| zFPQ>b%iAXqSM+G-tTeuJ_9G8&miJ7dvQwsa2`tZQ(q0BN(es%a1l@)L*lA0qcRs?- z3(r;mVy7NUisA`6<#rRsr_3R0!-;QZ|Mb0?v7Sk)-q}TIp{B*55vn!FcxDm>%i%y3 zSJ$37aQG2)up7~!?Lm4Mnc*c|)Wyc6?sKz+Uk`8g{6y(01;FC4Jpf>;QI!`eiS!l+ zHAn|E6N|*89p!NHdGn-P~7lF45RA5upC=8hngH%DN z?Blap34zF&HKRuGsZ?#08PPx2m2gA2ObN&T?6ve*K7zwGJzlOW?ivsdFgtk&vPVbK zPLQ2uaFHm6qS;^ahoI9g{_iv*ise`<%U;>PRH0Yj$4YM2+rg;X>iVD#rRIhrx#zjR4m2y@YBQo_zsotIPUekZWCQw zsscZN%(UvfsE+ycD1VNG39}zl0S^`SD+0y+e zT~oC~+T#x`e@Pg50!Hwa#wU6a=18btNH=Nb(Xv0_a}r6N#_7;`Td2|Xa4&jL=abL6 zDw@^;gS6OGEMQ-HaWf~23sc~tshFU^cd5|#r?;fZqAXOlsOjCc1g7_=LB9j4uCO-Te0sPX|T4b)abm0wOFaOD;&0$qg0Y_W zyAyqmu#z9MS2&&a9sCN2jiKV#FkGQH8rxul^X!>227exnPia2W?>(GeS-Uc?kC^c# z9Y=iJjT(dy3_M4#8jV9qvjj+^`1qsRULJz0>a^*zI@?6HyQkI~=*~eVy2}0~+^1;}hr=uF=GFp-tO(R5$Ogk=+Psi?1i_qv%bjw|Rfp-dB`h zi~60bX5n%6{llHv8YTcX$3wLL&AG%d0;D$%8&K#>I77Y(kX{jAi%>~s8IUZ9XnOsD z09qZ`0r^c6%w&#M^sG>)bdyF?Qi>`*&2%G1eDjPE5gAia*oe$`g&y*b1NoIJMqb!cQ9Zy7`wkGT(3z)D-VYM)dMDLr_4 zQe9Db-oDpGdz7$YY}H4 z)8+EuPp}mx+Ru67K6L(gS}eje-!_)Qud^st#=MjATzO{+)rujJPZ2{mX3xGK<9vBL z^KLi&hH-UI>_v}@PfrTb$E$E19Z)fO0aUc__m=FWU``Bx(l(BymGo^xc4!lj9;)gV zZUZ1(xCc+)866p%Izn%grQyrOeX%2gTjFR6Z^QvdX9Thw-u3~(^U4r5>#C4;?NG=V zxxD3q64JALz{4JO)No1y`Rc=yz>%SAlkojXPZoGwj*+`abCu%$tHOh24|~fTh}l!! zKNgec$$kXN_s6|(-kbf~W?qpOE|iWRlgmO>+HBdrLTD4`=He@vDMZ%BsG)p;6pNb7Sm65C(P-P z-9926|Gemw|D@53zt*Qr!vCwQxuhGD*z)f}JxizO1EqjvxyYPZCcbV*gBi=O(5dzf zV%1QBWiVpu6G~sLwFuuI#@ixBiNPdfHF07Kp4^lyoRvL&%H#&6!0T5M-PI8QMfZo3 zlO0@s0S7^}r7%(0=0GPKGtt}eWv{l^QB)a#LiK5#=4n0%ozxYkmmkZMyEIB#u7gpK z2=mf|x0=Eeb|+s z9cv-@^WUC8{eq5SnL@dir#t^$PgqImKh zSS`u123>Ts2+#|p_GxrJh7mAN#;d_#tpM!UBUc?UggwYNIQ_&pZ#G7UVwFVXJ&<@o8A#?3|gJiKF=st1VJ zstS!wp1X82$4oKAC}&opDo-j7NS*~wfB+~T>J7!vy3GRuARLnU)eN}}a_XOaPv8HH zqn?Vet~z)ddZjy%6gV`*wFUG7RN(uusoaoQLivn&0-@=4a4=L!DeW(oJn=Q-e-q@m z)g5e^>U`tgn@d9D=D+qDGKBuKly4Mg`05r6&!;t88x(>CRuzXoL?6e`FV#2;ft&e; zSKSc@#HeZ8v}PBb+D~_?g0erGJ2Q5s=w2UbXYhZ(!gmhd_ScF+5^$!dk9fh_l=G%$ zo8^)8)N^_lHu|GyeQCD)1?l^n>i9rm!7{GVqtI(*jweM1VU)Uo-^~3AoJr8IdV-#( z^X$VkkxdCX`s9bX4fn8Pp2c2EzggqvvOnW`yv^G1m6_o=%%GLv>Eztkh-^0=W1$+P$J^%m! literal 0 HcmV?d00001 diff --git a/test/functional/binary-resources/mocks/test-image.png b/test/functional/binary-resources/mocks/test-image.png new file mode 100644 index 0000000000000000000000000000000000000000..9c9ee6858821e1b27e709211f9e7089449cd51fc GIT binary patch literal 12192 zcmai)Ra6{J)UF42hv328LV&>scV~d$Zo%CxSa8?iF2P-bySoO02X_XUAcyZ?=kj0Y zTvT^;t-k15U2E@m@ArvRRhGd(B|!xM02p$zlIrjGq5lRl;`?=<4|oLta9GJnifMRd zpXH!=X%4kNyfxlWBS^)25Iu&-?euA?w+1ZA%d z(-~$h2y9=vU&%wT03m2!MA$$7?sgaL#$zb4f}ZE5JaXotj))`NSLzzyhqF0}EW5=XyS*Bc6Ta5qd6G-$oH$c; zPle0TCh(-E%-yP4g4z{E7bEI(Efj|J7<9BDY_&1xtF#j6c$$k!ctK8CeqBXjy$2b| z9;$?}2uhiq)(BuXFe*quzlf@CxzFlYSDLkP)PtV%r2VaO$ZQ>i=cC@&Z&N73EtvVA`anVxf> z48o+U9NAr;JLu+$SzKUfF-O_QjzrWXNN*D;^6RGz=7%o|R{AGE*g(3AEpxgK%HWhS zc)9GAJ_2C&8E<{_C(N;x+Cp$bM*J$fq2S(j4)2U?DE!w&?_O-2X^;fO2k*LuDEHKW z%%d)%ht!B;1nEdpZv~Z$MImaiZLU(aeo!u`Z#ENi3Q^=Mo$eikDw<#=;^p{`+0!58 zvnJGA7?sJg>Y;BWm>1=HfLBLNPr`!&O$s$cNOhQ;epWyu|4i{{iEyzmt@TsIWs0ex zD^@m!RZcys?(ph83LwC6S|)7gMrFzq^3_0xK@8@VQEK?uY!~j^r89xd*T2zlw@-Hq0uG4)uDX&T4)b=l;GSiXW*>ZpK^ESf(&F77D}0ts#O;7E36m@x<*) z6g@|>eaztS=&=3`<;mGMkxQOTaz{-spp4?>n3_5kxpx<}xE$f)f%#xAmQhB{R_bgq z7JHN#DB~Z9rSg%#h;_E5%hTEokM=GeW1G@bCe)rXV%%G1n$|mn(q5*tf$>H;*O1k8 zYJ_*)haPWz&Y_uN9z$g3q!Tq5M;Yh4g4TUS3QXR7qAxjTA5Wy2Tz{h+Ww39Myq_It zBCDPrp1jppl40@8giZY`CXO1b47-$Sa$(xpEE5kdaz0I;aJes>A?Pl-p$1u>Y#=49 zwv8dikPyM)1d=J@pE;YI`G<0$7{{z7f|~t*Q&s+4n+2sThp28{VR>-vIvr*T;r`%z zI#^afJZ^?_6Pq32$%GK5cbxsLqv+vn%t~|7J&P4^yJlGR+{t zXO-Q-6FU225t>PZou3&D%yL7_gZ@-UZz4}=3Tji@td~uWy&XGpo%X^g2gCK7MJ7Q! zVVlC#=kckOlViDV?}?rChK`+W0yc5)AvKO?O76A9QAnFp^R(k=FB0|cbO}CBE+m8- zd;(mXAGum#q*Cg5PX=L0Aj#doiz`}bq=Niql{2EqCsuhnpe>D;NS#O^WATmtJh|)N zED|!wlnZJdIrs%AQ6&_+Du9IHRtXbRKk1usu6ZS;g+Drvf0PorYi8n4{p9kY7^kR8 zeY9ERM5eCBSG`WFbR$Nnib$^e`fTxQCf zywIX7lNX%V#8s?9Rg96cy^)NuxOPVc z9b}Fl%B%fVZ_c9r*h;cs-u3aK#aChS)n6Vwf9$*wE`QBM#bNy!%I}lezM_fmk5EgEkS@aW zc7Y>C=10|NMgU3C-!bhQ+R)*0WmLbJF$L`aY5 zwiNm|`}uuzMfke`x`a2K_O6>|sth2s$W)%N85ma#y6r}l&u}N;Usp$PMKI4FdU>eP zWSoa(N>Oq;{i6?;&pV#~a_mgrva`zSo!=288u({lS7(B|o2*8FcODN{#QpAjel4qy z^R5ve(kXTIL#akS#@jmn!-~lgAXlcvZF85-ruNvDU^Vg~%zYZbJ%es|r~2fZO}Bk|X0m&``0ugW=cjO4w(1@6pkJ*V(XRLsaI0MPZicIQlLd+x|;UeD~? zptr6&d{41E=KfMKGx+K#n>BD(=bpgU+;fNwaJ9jL-|X!I^ZD$4#_Xhul(Xv|jc=!k z3$~fP4m1?!`TKg0V=F>rIdEo$N{CONZ|J;|5K&CYwJ#eq^dj;C`uO}5Kf6G1>l(q0 z>2@(h@xdSK!S78kTvy%LA2o0_=_VZ@MBX2rTSJuFQ41pioyYL<9x4i;nEASZds4o( z(%dC1{CJ_nH4D8ldWpKOfc1<@ETBli-9uTrsaENeZkB4@j43tRATG})i1pn0vem(6 z?V8dJc4W2LVDh^-egb?%{3Ka+W^mavU=`cd>9KDudO_D&3m)9@8U8uw>pS;Y<(&zar&2?Bx0Y4c&Yqm`LsaM$R;J62#sSvBh$d^Z3Cx^MuS;&x8+w>q{zpc#9 z5km3Nmz8e!+VJ@(dga!?ny@C`I;Iv(E4}n;+xu69Z78b5v!WTuYvnWbL40gj|FQlj zJv+zjG{S^*4MV`2!sdjq$-gA5hl{x`{g=)oNG8LX;opeW9L#N9JV;Y(%}AvPkI!>h z&5a(@==u8Y%a(P++zjj%qLcbxjxcLkCx3L*>RXIW&N4$@j8|8shn<(!t=V>Nmh&v& z_L9}271}+y+ueHr%A6}Znxp~@+@~7>Rb?pb+Sk_Vxn9ygwq!@9fRTBM_{ZLQ=59XU zU#ylcsZuZx?TvD;j_y*gDzLI^7cwFBreSkmFruy4ZFPh>zS>=_ItC8st+C9Gb%(To z29Iv?p+uq^71Nvwia1E6G4<6i=P{r{^__q7GY`1+S12Re4;u@st-k3FdM_CV5QUn* zMb+_%v)_^D8Oi-)mL@2UMbwE+|1AT5-9K|RP+W2sU={p~@3TUdw|ZI(#0eUJ<(^3v z_V+aukhvH7J}=>DXuUz)dfHXpxhnk|HyNOK1%=z998PsmWFU;GAS2PPgYW0;FjEdY z`}m!Wv*Hx7dO6w45zEl)E?{e>zNG<}#rHG}O@NeOD<_hX9Wi^%_yHfJcBM+4b&UcF zV%Mt1VHh@7KEIjyyrlPG!T52lAYFXj)FJ-{mg$>8E{jaB`}(Iudw$`7wccI&*~}l) z@NFy;;%L~}VbOQS;)vpc`6u4m*?H!HLLc%Mr7YJl^SaDKm79OCqSalx%j7(_p%Axq znyOr3C4AcrH=ha9Gjf0LQWwyJ23=%Aiw^6Jbq}C6b+db^(z3_qnx|y=9qjU6EJej6 z98u_A6z$KnHZJCU(cVJhTYHEYdVLDtsr`{!+dx-3#4#TmG}n$0ULEWCcZ>;$51l|X z=L;matPQMDw@+}OoRSAz+8VOJ>6!hptM~h@i1dkD#XS~LHN#qrOR+U0BA%sd4iX1%1DIQ%1t~td}Km ze|`)b7(WflGGpI57PL(m)J5&H=!|il7{%ZVsGi#&CdSWPw+9=5QZ2iWXnosuRG;Be zCfL%tbSA|S_PEkyda>9%^O?p0XDqYtoATCsa8k}CM%xxv@W)2Fx)dX^+I_lmsjsLX z+7aL|M(sU>AyeB1X4B$|qq~*D@hz*bT>`Fz65z% zQg)Nn{d$gd&})x;?M*Z&;N;{GMS(q+xJc9e_=jW|+dR3;OR%2YuLx!Fd8-eB|@Xa{?YS>L)#GX%aEHUd}yt zzzwaU3;EQ+<$2{q#zj2}YTQ`U8=ky0Hu>}uf8W;HrO8v>FY59XsOtRjqOtcYR;xW$ zyoS0Cclv?Gx&B9nZsgBLdq&9qG5Kmuzp&gJ;?ewL@`UC-_x(+Tvkdu8acuHYoI+xm zvp750*IRs^g=MdjW8}JQ)qmy#Ee#?mCkd~IH3OwnW2#RuNl~JoI4NCy(gIq z3bNuDeZ{utWWLG`hJcj9nBNUp-OA1?r_(lTIKfjNfZjz17Is@Cb{fk#7-N>$j8!cg6D+&`u+M^+ zx7NSHM0O%uOC^)b^Pw#MR>;sEz9rM3^JL7}g3*Q?d5&a8$3^MZ*i1BUcavAwd)kCF+wHFo-}T+x4I!VybXQoYb@VTsdiTw{b&k;h{pj>B zLg8`tm&b&nYNgcIsY(+Coh-m@;A&>oueXo6>)dz*{;1^qO^Vv2xgD3DV$oycgDm*| z%VYkPU`v&PCPWIsgU|Q`>E-%`gJ~;zbq5gRa3c-g+JzwcN*U;`zCTY|G9KmJ$9d&T#i+?iHd<=3cN{nsjA;3_Vv zyz+piUqc$5WP}mZmk}V)W`=Ui{o8nv4rZ^6lm0UBJ#KsP?fB*UMkttMub$EZ>%NJR~lpNbVom)jZM6H0D%untT53oEzcaui3t zAL)xlpiRNnbGY)=UrMwDL%19ybqlA^PMeM~-RXvim`+~pXTUy1AUlPhV#7bQZwrbo z7|APlt-u%D`W;1nX>I4mq!yKZ9w#;5{U(^D2j_TqJ4JeGD0lj^rY52Z3_n+9%mvLl zHJoc-OfO8r%@%GqVt3&8vs+(blEhR!fJdN_)+}@hUS6a}Abm1?o(LIVX{W8Zqn78J z?KjU6V7ju%qE~Mq`rg4lBgkSI9af3c_+D4+myBcyQJ`cQ`#}!)F80AGwKTl~B6B1a zu`SUX9D)s;Pd`qJ|9n1aFg-SiGNE`4CO1S@`mPLI*Y#l`#gkXwYERxHPqEHYTv~cTLX?7?Xr6=$|&%*-pzDh73Sn(`C4ne#i2be;nnuFl~Se9~Ig zXsufM=m5jVhLiv4n>-Mbo@MuaaE>P)U=!30b=i_#!C!ATjtP!Ry_pbfkvop#CrB|3 zh*K-d@h*k4+V5ZcGKp7|M{s|!anT%lV$K`4B+}WSuQ$&eLJA-=%g~l`D*B@s1f!h7qzo#emL@ko{YMTIs&l;ziE0LXb*`dXduR zNA!`lzr}1VMQp9{wXC6TvnFB=BR{lUR2)e;8XZker4Dq%ZJUA*%6ijMTO-!EZC|?T zt)r_$2}++jhx}xI!zebb-dCjI!USf}Eh+Hha%9-uX{KvSukG3aI$4`vEC3@qc7h{m z>_LQB8Du%1`J~Npn_#k(dW@NHu7q`5EZxn|xD?gxbbg#8?kobmu!9J6{)ltp_I7@- zLeu*SUQIgA54*pI9ffconb#Y(&V9+DmWaXt>ez6RGFFwwFyX6cyP@hO85`9t8upBI zPWA;_)c15-7t{{xEM~Rw8hQKNw)+J?nzk@DI2BDu*65Qy_Z?n3UyILtSRvJVGMSi| z`R~OqldEq|*^GbI*GXP50jgM8lhbr*lBlgw2gMX6rfbGUwW|@JA!8$QhL2vW)~Bz2 zmbo(ziq~-?dosynmXB)N=nDQ#0tXl#?52QXsW}2F)0R)oSXrjW!kBwj0*5}BbaBnB zIR9)Vs1NM4d^g4aVKU1*=h&+uTM7dpstd1m&9DAx>!Z+9Za90gS^UV>ww7#DtROYS z^(TvZZ{E#rLY~}a5xe>xhtIQL_So2SQVLy_5-tz$BoVK-bXoK4<3vpIv$I%*Tb@&g z8$GfFkm#v_wybP9tNBff;_du%0qVWrT|7_vg5vOCtt$;acwZDcHp(WtXrNdU&7dWh zQz1EYTr#K~@X9yv7d06CQ_k z9#^B0i_>o3iKRThx9TY%ye$ybUdePHpI^0AYK3Y+l~4@@D3w8p&`(BON_u()}%yV7}E z5hZqV=}U_y{d{87i(sZ<0whiJL}w~9R-pgRCx*4OwUfBIvMv~Hf3D~C9(qq`=VuAu zJstYCqvRE-cRkqTy2st(@w_{(*KKEkMgM45bI?SzIB)sm#Y)`a8C)&PU34DC<9BctD2JgEzY%_Nn5&> zFUV=L)>Qn={zDfOv$nf#mg4B*GI3F{S|0RD!Fu; zydqgB1>_Np!$!)5?krge{;?m~T6vz*(o2hKq$hLWLPWHnHT2sc(a!7VNbg}Dlym*W z{6|N7Ok+~Io@YiV@wM}89(DwFcCscl>y|z1+W94aiP=W}tg#|%oW`8mdv1|W#bYlj)&qyt?@JcHun4c6-v z#Sm#mx_g!pD)Bkci*PFmNg;=y<^go-*#Gd zlLd@2d!hKHq=R{K_oV_-gaF$y9o{Y|b&J)JtodA7+q(42UYWs8B1l6DHUG_u341Gc5L%c`$cl3tBFN?5#kopaKBTq^-ZxXx-nS`)qa%3SJM6fUZ1vD z43Dw=flpth-xyH~DMR-lYquPoBocALe?-xQ+cWJ$)O8aKMM|D4a!WAoJtnBw)Y0+F zjD?@us{2=jAMo$bCW&>_XvhW_ix>=JOlXuZ9(++{dki4ZvpO6GlbfYww+R(os@vQ; zjFu9e;A{dg)fmFq0b8B{NfKj7!x9@z6Z98T@#&q`)uAa9~ zQAJ{7CKR94P(SS9&$?Za;g?@^6G;5_x~3{(iaiZylep1My8E;Z;tEl;+W2#ea}P~- z2f@VDbJW?wOcMbD7qSe8O+s#^aexNKOyv(L?yjUvV#Dk8jD zCA*h2SpECw^)xoJD$NOzS~MorVJno9%JA5nT6?W3f^2(Z{BFcg*Hv(GmY4@I;_lwe zi1>(rptDf{e_AH10xKR@W0=qTOB<=1=jc;NP>Se9Yq&ndOtsuhchp3`H`T}|M_F|3 z&5qsXiOrmDGR=OyF4h!bQPmvtYLoG}5z!k|Gp4i5d>tD^oQ&w`1o=2XP;D>iFYr`y zg-n2Z>5Doai1pH3P%UM}NQX+dYcgo|^p}BpovneYtkZ@PwET4quY3>lEL{Pck!`tT zL9-*}WBSrKyZO-dtZlyKKRO#p=uB^G&m4=52gwNf&Lo^=&wxg90)KRl_eMCaURnUH zU}wWGzC&?@ z$XkmbJie&_yEl^#G4bd}s)`19CCYBt$9cxJurfXo#WP~ur6Pf@6 z@E8ynbcMX=*gj`2j;P|7o)zQYT9CtbST*5^*<8T>@H##q*OVU{V2@lmwVsU3=LUYE zs8|4@1hC?M&KiZzAyL7%zWFg>oo_>t^C1CQ0tO^{o=Z3h-uOflX*_7QwS%w1xmb?D z4Qh6w_U6o1aNF8B-&vkzSz{;R3f5qSexuAPZnLTimyti7V~G@!9J9G@Yv9T2?S@_~ zxrHhQ*q`>svxSrK`rY=tJ{Y`tEPE##HRdPo$f;-TkaE$#9d)(x=HcsXmXnyi!`EEK4S%wvNy z*P0qH_yqn`#6>D#m&Py%Zulm?;iC1B|7;mFS@~V42=R)`tMT~Dq|-d$yms?L5q^v_ z<0L*d_(C+B?GHZFKO_=8i{LZVUas%Gud3_|O@#mHTLP>o6$CmSTDmEca%Z zhFKQ%yh)%&E{-yv#bO5q=%zoNCh=%>aSJ?aIhga^{yRzW30LvUx7oDIP4(Nd*q?c% zY|3*O@E};Ho-x>D5+*_AJg9aj@YB?0W;w0$=0`}xywR9(l8Z54QD1Z8(hqz zz6wxM+ah=U@RRR*14!zc(vuFb7CNC%+Dr4h4fDc8CXKBzrB6JwkwJ5B@DBSrfEL1! z58-r4TKs;sU`6E9AKA7&{^Q#&Hj&#Sj!LEzvZca22@?vbL9g9%6bONX`=mx=nn+en z8pWG2+9{gbE*xsIP^prNAqbS^9R4p-F)}CWWip3gR= z`al|*yUT-AZ11jQRZC>da4$92B74Dz51+)gF{^W8SB# z$nlOd_mBNN^IpdikIwo-6-2yCq4buYhLzjV%i^=VIH}$U9!M`unTSaoPFIWZs`4|83!Htsh zDks?IFr$*lA2U2Jdn^6-9>?|Ma~UR9SpC%+SxlA(LSR8UNBEcQv{FdAbegY#D$%1d z%n32Iy1qb!obQjw z<2FvUx8C_ORf2C+`8pyq0(sT9_IQyWc9KpVIW_gmVFXgas2hLsfQ}#&<8&K zgHQ3HQ?r_4zr!V+{=)Wra;b>De}^GXF%(VC-(izwSFvTLHFH3eg1qyw)n9Y+O!TXj zSPtP8w)d4z)yA7%E=AeG6(nGh1Ao4xpW3nPD_VoN{X9z@aea2_YjyipOV%5!YI!92hgb$f zzl0(rikI?`3oZybJ3J48&xg(F#+TDo%xVIDaJ_*gD?L9kDIuudW`b+zCMQg&Dq>hQ zHOvMgqKDqHu==d?Ngu5W4My@@1iG)Y{;7fdLRaz#m*2qzonSQL2>KtgI~+))*!X1X zwgs#O7{;1;R8kZ)Z0x69t-w77ql%%H`&K25TXIf?JImLC|2&}ou&XkV%gBnaeI6jzX!F zVpBvrWgN)pu5IMiv8ES*NQr;_j=5NWxBaAVeyz?jjOS>(zFDtX?c?H(j)8lm4|+>` zsv~Ou7yN-~$Z%I+>S#y`>6H)2i2+8rjCeU!?DXM{|HfL%{^5h|^-Ju#q=_e;NRze~Z@Ax(pg9|2CX7<@s!_bu8_abuFCevz z?$5KC!J-KrKl~K*y&uonwKbxQ*b%i11tv<;h_i@-*%CJ0er<4pZ~4pRqug_NXy`NC z^-Q-uGRm>c|ATUYmq#4D^_AYp6dDXlV*bJB`x{YQ&t`_q21;Zrk_3f^FW-jTE}k}p zE!F;1$_I-ujw8#uHv}wW{f~8X>i8cfSMi zTi|>)3J4bo@A(g=JAP)|7g0UNp+A&RN?nf2w3dsG)F9T;%yCsPsVL${N6Bod0R9SO zG5-}YtAbeQ%N-RwJ%k^cuw+F^ImNB~H!E%c7bY^A_Kr$d|74oie_B_JrX)6)y{*Ef zJjt{w2~1lkaV&v5oS-{k_zop1ZV(-3C8~)yACrM$Y$ky2tm<;#^Ma;13D{W0S5g}s z8~aS|GY@W+7H8l-k{%7s*ZhL+aRE6G(S!EKs8CQZN}Xs`7}AG zQC$!}C*w4LPlD&eu#Im9ja|WFt*BdEF?DI7> z+0ap2!FQl>t&Y8g?;8!>dLjxha>+ut4yAcwH%JN) za`uzL)g2LW*{bUD9m))|y+cHF5EKJP(335AwxrFMeKN%shf}TvM>H7AePA{hpu=ma z0ICZH3KZ9<6u2q@-MXWXxM-l*0uMhDN75*hEM*bG1RJsJ^@Tzsub&~k2U85f6fB+C^$pT zE=%5#mAQ5%-wDp5z(KM@@rMMeXu9|@nAVx8ctYctMMvYJ;ypDd|1R~<3060K|E+93 zDe4Py&Fh-Gb<~5Lxr+*P8znzkOgES`yOWQ8R;eTFGH!q!Si&Jp&;MBgJ{{R~^bjUw z{gOu-5r0&SloJ)nwjWt+jdYRIh%`QM@rRquhA`cdJUCR4`F!(P^Wrm2Y1@2-_PH0<@Vf> zIF9lgrODI;$eR43KhC6G<~EdlLJ7rZ-}pgk#oS>!{mVFxSdpU8!|mx;P+a>b*&`oS z|Cms*ld=Iq8}aqHu?zCf?;R^#zfFMnl6mh(ci-wjF1sRvm#O=}L1yC}*R5c+J=ZQI zLNnK(pw0qGJD~cGjuwMLM{l|%#VA4q_s_;%;N%tkW%9E$PrfY4s`;q&p;lLu>9R?q z_wdl)9)-OF9pn4Xvdiu48(-CJ6{CjU7y9299Q2*7vK3z@iOBe9lx*DU8niQA8b25e zt=o)KQGWER1Epp)zvB$Giu~|{F@2{BxaVXXB$S?5#Y2N8x!u1RX4w{|6OgrC9&~ literal 0 HcmV?d00001 diff --git a/test/functional/callbacks/callbacks.test.js b/test/functional/callbacks/callbacks.test.js index 1e07e916..7f6b3279 100644 --- a/test/functional/callbacks/callbacks.test.js +++ b/test/functional/callbacks/callbacks.test.js @@ -6,7 +6,6 @@ import sinon from 'sinon'; import scrape from 'website-scraper'; const testDirname = './test/functional/callbacks/.tmp'; -const mockDirname = './test/functional/base/mocks'; describe('Functional: onResourceSaved and onResourceError callbacks in plugin', () => { diff --git a/test/functional/circular-dependencies/circular-dependencies.test.js b/test/functional/circular-dependencies/circular-dependencies.test.js index 428669e1..d5a5c229 100644 --- a/test/functional/circular-dependencies/circular-dependencies.test.js +++ b/test/functional/circular-dependencies/circular-dependencies.test.js @@ -34,10 +34,10 @@ describe('Functional circular dependencies', function() { ] }; - nock('http://example.com/').get('/index.html').replyWithFile(200, mockDirname + '/index.html'); - nock('http://example.com/').get('/about.html').replyWithFile(200, mockDirname + '/about.html'); - nock('http://example.com/').get('/style.css').replyWithFile(200, mockDirname + '/style.css'); - nock('http://example.com/').get('/style2.css').replyWithFile(200, mockDirname + '/style2.css'); + nock('http://example.com/').get('/index.html').replyWithFile(200, mockDirname + '/index.html', {'content-type': 'text/html'}); + nock('http://example.com/').get('/about.html').replyWithFile(200, mockDirname + '/about.html', {'content-type': 'text/html'}); + nock('http://example.com/').get('/style.css').replyWithFile(200, mockDirname + '/style.css', {'content-type': 'text/css'}); + nock('http://example.com/').get('/style2.css').replyWithFile(200, mockDirname + '/style2.css', {'content-type': 'text/css'}); return scrape(options).then(function() { fs.existsSync(testDirname + '/index.html').should.be.eql(true); diff --git a/test/functional/css-handling/css-handling.test.js b/test/functional/css-handling/css-handling.test.js index afb3cd64..211b2b1f 100644 --- a/test/functional/css-handling/css-handling.test.js +++ b/test/functional/css-handling/css-handling.test.js @@ -21,11 +21,11 @@ describe('Functional: css handling', function() { }); it('should correctly handle css files, style tags and style attributes and ignore css-like text inside common html tags', function() { - nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/index.html'); - nock('http://example.com/').get('/style.css').replyWithFile(200, mockDirname + '/style.css'); + nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/index.html', {'content-type': 'text/html'}); + nock('http://example.com/').get('/style.css').replyWithFile(200, mockDirname + '/style.css', {'content-type': 'text/css'}); - nock('http://example.com/').get('/style-import-1.css').reply(200, 'style-import-1.css'); - nock('http://example.com/').get('/style-import-2.css').reply(200, 'style-import-2.css'); + nock('http://example.com/').get('/style-import-1.css').reply(200, 'style-import-1.css', {'content-type': 'text/css'}); + nock('http://example.com/').get('/style-import-2.css').reply(200, 'style-import-2.css', {'content-type': 'text/css'}); nock('http://example.com/').get('/style-tag.png').reply(200, 'style-tag.png'); nock('http://example.com/').get('/style-attr.png').reply(200, 'style-attr.png'); nock('http://example.com/').get('/css-like-text-in-html.png').reply(200, 'css-like-text-in-html.png'); diff --git a/test/functional/encoding/hieroglyphs.test.js b/test/functional/encoding/hieroglyphs.test.js new file mode 100644 index 00000000..3081df2b --- /dev/null +++ b/test/functional/encoding/hieroglyphs.test.js @@ -0,0 +1,41 @@ +import '../../utils/assertions.js'; +import nock from 'nock'; +import fs from 'fs-extra'; +import scrape from 'website-scraper'; + +const testDirname = './test/functional/encoding/.tmp'; +const mockDirname = './test/functional/encoding/mocks'; + +describe('Functional: Korean characters are properly encoded/decoded', function() { + const options = { + urls: [ + 'http://example.com/', + ], + directory: testDirname, + ignoreErrors: false + }; + + beforeEach(function() { + nock.cleanAll(); + nock.disableNetConnect(); + }); + + afterEach(function() { + nock.cleanAll(); + nock.enableNetConnect(); + fs.removeSync(testDirname); + }); + + beforeEach(() => { + nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/index.html', {'content-type': 'text/html'}); + }); + + it('should save the page in the same data as it was originally', () => { + return scrape(options).then(function(result) { + const scrapedIndex = fs.readFileSync(testDirname + '/index.html').toString(); + scrapedIndex.should.be.containEql('
저는 7년 동안 한국에서 살았어요.
'); + scrapedIndex.should.be.containEql('
Слава Україні!
'); + scrapedIndex.should.be.containEql('
加入网站
'); + }); + }); +}); diff --git a/test/functional/encoding/mocks/index.html b/test/functional/encoding/mocks/index.html new file mode 100644 index 00000000..8d724cc2 --- /dev/null +++ b/test/functional/encoding/mocks/index.html @@ -0,0 +1,12 @@ + + + + + Test + + +
저는 7년 동안 한국에서 살았어요.
+
Слава Україні!
+
加入网站
+ + diff --git a/test/functional/html-entities/html-entities.test.js b/test/functional/html-entities/html-entities.test.js index 4087628b..cb682dd5 100644 --- a/test/functional/html-entities/html-entities.test.js +++ b/test/functional/html-entities/html-entities.test.js @@ -21,8 +21,8 @@ describe('Functional: html entities', function() { }); it('should decode all html-entities found in html files and not encode entities from css file', function() { - nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/index.html'); - nock('http://example.com/').get('/style.css').replyWithFile(200, mockDirname + '/style.css'); + nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/index.html', {'content-type': 'text/html'}); + nock('http://example.com/').get('/style.css').replyWithFile(200, mockDirname + '/style.css', {'content-type': 'text/css'}); // in index.html // /fonts?family=Myriad&v=2 => /fonts?family=Myriad&v=2 diff --git a/test/functional/redirect/redirect.test.js b/test/functional/redirect/redirect.test.js index 4a25dc63..a7e4f16f 100644 --- a/test/functional/redirect/redirect.test.js +++ b/test/functional/redirect/redirect.test.js @@ -23,16 +23,16 @@ describe('Functional redirects', function() { }); it('should follow redirects and save resource once if it has different urls', function() { - nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/index.html'); + nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/index.html', {'content-type': 'text/html'}); // true page - ok - nock('http://example.com/').get('/true-page.html').reply(200, 'true page 1'); + nock('http://example.com/').get('/true-page.html').reply(200, 'true page 1', {'content-type': 'text/html'}); // duplicating page - redirect to true page nock('http://example.com/').get('/duplicating-page.html').reply(302, '', {'Location': 'http://example.com/true-page.html'}); nock('http://example.com/').get('/true-page.html').reply(200, 'true page 2'); // duplicating site - redirect to duplicating page, then redirect to true page nock('http://duplicating.another-site.com/').get('/').reply(302, '', {'Location': 'http://example.com/duplicating-page.html'}); nock('http://example.com/').get('/duplicating-page.html').reply(302, '', {'Location': 'http://example.com/true-page.html'}); - nock('http://example.com/').get('/true-page.html').reply(200, 'true page 3'); + nock('http://example.com/').get('/true-page.html').reply(200, 'true page 3', {'content-type': 'text/html'}); const options = { urls: [ 'http://example.com/' ], @@ -79,11 +79,11 @@ describe('Functional redirects', function() { ] }; - nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/relative-resources-index.html'); + nock('http://example.com/').get('/').replyWithFile(200, mockDirname + '/relative-resources-index.html', {'content-type': 'text/html'}); nock('http://example.com/').get('/about').reply(301, '', {'Location': 'http://example.com/about/'}); nock('http://example.com/').get('/about/').replyWithFile(200, mockDirname + '/relative-resources-about.html', {'content-type': 'text/html'}); - nock('http://example.com/').get('/style.css').reply(200, 'style.css'); - nock('http://example.com/').get('/about/style.css').reply(200, 'about/style.css'); + nock('http://example.com/').get('/style.css').reply(200, 'style.css', {'content-type': 'text/css'}); + nock('http://example.com/').get('/about/style.css').reply(200, 'about/style.css', {'content-type': 'text/css'}); return scrape(options).then(function() { fs.existsSync(testDirname + '/index.html').should.be.eql(true); diff --git a/test/unit/scraper-init-test.js b/test/unit/scraper-init-test.js index 9612e180..080c5976 100644 --- a/test/unit/scraper-init-test.js +++ b/test/unit/scraper-init-test.js @@ -121,7 +121,7 @@ describe('Scraper initialization', function () { s.options.request.should.containEql({ throwHttpErrors: false, - encoding: 'binary', + responseType: 'buffer', decompress: true, https: { rejectUnauthorized: false @@ -143,7 +143,7 @@ describe('Scraper initialization', function () { s.options.request.should.eql({ throwHttpErrors: true, - encoding: 'binary', + responseType: 'buffer', decompress: true, https: { rejectUnauthorized: false diff --git a/test/unit/scraper-test.js b/test/unit/scraper-test.js index 87e566e7..87ee177c 100644 --- a/test/unit/scraper-test.js +++ b/test/unit/scraper-test.js @@ -103,7 +103,7 @@ describe('Scraper', () => { rr.should.be.eql(r); rr.getUrl().should.be.eql('http://example.com/a.png'); rr.getFilename().should.be.not.empty(); - rr.getText().should.be.eql('OK'); + rr.getText().should.be.not.empty(); }); it('should return null if the urlFilter returns false', async () =>{ @@ -138,7 +138,7 @@ describe('Scraper', () => { rr.should.be.eql(r); rr.getUrl().should.be.eql('http://example.com'); rr.getFilename().should.be.not.empty(); - rr.getText().should.be.eql('OK'); + rr.getText().should.be.not.empty(); }); }); @@ -160,7 +160,7 @@ describe('Scraper', () => { rr.should.be.eql(r); rr.getUrl().should.be.eql('http://example.com/a.png'); rr.getFilename().should.be.not.empty(); - rr.getText().should.be.eql('OK'); + rr.getText().should.be.not.empty(); }); it('should request the resource if maxDepth is set and resource depth is less than maxDept', async () =>{ @@ -181,7 +181,7 @@ describe('Scraper', () => { rr.should.be.eql(r); rr.getUrl().should.be.eql('http://example.com/a.png'); rr.getFilename().should.be.not.empty(); - rr.getText().should.be.eql('OK'); + rr.getText().should.be.not.empty(); }); it('should request the resource if maxDepth is set and resource depth is equal to maxDept', async () =>{ @@ -201,7 +201,7 @@ describe('Scraper', () => { rr.should.be.eql(r); rr.getUrl().should.be.eql('http://example.com/a.png'); rr.getFilename().should.be.not.empty(); - rr.getText().should.be.eql('OK'); + rr.getText().should.be.not.empty(); }); it('should return null if maxDepth is set and resource depth is greater than maxDepth', async () =>{