Skip to content

Commit

Permalink
Fix issue ExpediaGroup#199: origins.ID.requests.cancelled metric. (Ex…
Browse files Browse the repository at this point in the history
…pediaGroup#209)

(cherry picked from commit 4123874)
  • Loading branch information
mikkokar committed Jul 10, 2018
1 parent 6a023ca commit 5cf9283
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;

import static com.google.common.base.Objects.toStringHelper;
Expand Down Expand Up @@ -147,11 +148,17 @@ private Observable<HttpResponse> sendRequest(HttpRequest request, List<RemoteHos
RemoteHost host = remoteHost.get();
List<RemoteHost> newPreviousOrigins = newArrayList(previousOrigins);
newPreviousOrigins.add(remoteHost.get());
AtomicBoolean completed = new AtomicBoolean(false);

return toRxObservable(host.hostClient().handle(request, HttpInterceptorContext.create()))
.map(response -> addStickySessionIdentifier(response, host.origin()))
.doOnError(throwable -> logError(request, throwable))
.doOnUnsubscribe(() -> originStatsFactory.originStats(host.origin()).requestCancelled())
.doOnCompleted(() -> completed.set(true))
.doOnUnsubscribe(() -> {
if (!completed.get()) {
originStatsFactory.originStats(host.origin()).requestCancelled();
}
})
.doOnNext(this::recordErrorStatusMetrics)
.map(response -> removeUnexpectedResponseBody(request, response))
.map(this::removeRedundantContentLengthHeader)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ class BadResponseFromOriginSpec extends FunSpec
it("Responds with 502 Bad Gateway when Styx is unable to read response from origin.") {
originRespondingWith(
responseWithHeaders(
HttpHeader(CONTENT_LENGTH, "0"),
HttpHeader(CONTENT_LENGTH, "0"),
HttpHeader("Accept", "ozogxzNsalwzakvvbBigkjxkipybjmnrnDriwkniaKixkbornvvxdnavwCstDVnfubhJusejtcndrInpluz"),
HttpHeader("Accept-Language", "aktqujsfbaSfssuocxfvtmaumhmbsjuupdvpqsEvmxqkjIsqeqxjabqnFWmknegasinoparupLektbieljegxaMsprqKkfhhby"),
Expand All @@ -98,7 +99,6 @@ class BadResponseFromOriginSpec extends FunSpec
HttpHeader("Max-Forwards", "jwwrqibkvvqfztAunzyzHdkjlIzSyvqanwotwkbnskgdydBdwscpaPs"),
HttpHeader("Referer", "gnhmozkdxpwxalLqBYolchqmuyagRcrrOelnCjnzjJfBfevzhvrrUtdoixewiqdf"),
HttpHeader("TE", "dztchfepqlfjwbontklxwbnGsppJpdfyxyqtyniczuux"),
HttpHeader("Content-Length", "0"),
HttpHeader("Via", "9/1 fwdhptykzldheeyfycdvzdruomxjkmujefrzonvxmjrgfqimsanxyqiehwmolfncBdtlkdMyLhtisfvFhtwdXbsnzuvbgMoi, 0/5 sznkftqdDr (djgpydxqcmdotgfphrjwwkydntamklvdeqUldCeXiGygTuxxunmabdiekiFotvSmwofzmwZg), 1/9 bzcmjmqgxkelpqfaQ (lxlvu), 1/7 ywdlKtnulOzxxpwhxwyjllylxeqdufknfeuqcimhljosfiAzsustwhyOUazjqQxkxzvvgdlbyhioueEXFtfkrkfxvrypjrruyto (yEKzcshfbdgdwybimaincuniziwohsnjcrMzzsuluiybctsmoxxspDjdoqgqoqmhCyjkftlyfNuetboebzpquzveyfjlq), 1/1 wckupaglycEpzngkczofbmhujkalcxdhjjjgqsiuemniyi, 8/3 yhtrrPdpjsccf (rptiwedDqjDkimxmfuwlgayabmbqbwxqjsoveaYrrndkbegpdjJxvuoiixrdliefvsmdvgdhx), 4/8 ijyLFwcVssqsuxeknmpT (fetwzuxhcdhdSrjkcjlanqmufBvssoemj), 9/1 goccnydlmczxoUxLfohswfujT (eFiatihsawCacckujelbbbimlcsutRfbeDxmbJjelcexqX), 7/9 fpvsvgzdggpnpkbeqkgbwshnqsUszyesrwnaVyrrgqgtdysiClrikcrywzsduckmzcpgZquuqlfsptpcJprc (anLugfmkbjNVkzsusjgxHpeniigvpEUWfdalbpqnrflbswkezuvhihqlvwszbwgxhvmwmocpmrcFtvjnqyosWpixZ), 0/4 ymzriduxyzjsdtbuByjhPtpesafpkwqnwDotdxzcpdwrtsnmggoyulEtkslzghpudxpwwxnfdefIhgwHxe, 1/4 mawrokoosxitpwxLbZochwkfluyrAj (enoqmoaoiMpjwaszGifxotuxeqofqvwaOpvyttedvtbrmcxvwbpejkqwtojjaevlvsRcwjJpJKcxsnDchjxytuphibzwrrVlxq), 4/8 mpwafTnirMbxbcnujdNveivnkwkhivfpcjxkApxuYxlwolxzkkclafCpxuyxrrvglYffpqfgmgloudxbfgqxjyugkjUluxh, 9/1 sjxbxbgyxhofavayuurjoxJcfptbrvwWgpwRgvsIgyw, 7/6 wexgodcotwrsisihiayskcvspuFpmbkgyfkrvrfcsGfszovfoshhkxinwtxlfkwkem (dgfidmxvzwxcEswjBhFjDnwfxbBvzwhcrtfisxftshOtzuuasteyugRvzahrvZny), 4/7 oqzlzacfkwj (yharaitoyvrtamgQrKohvlrqHxhxcnkonhqfbbGamo), 1/6 nxpyDgafbbcqmVdwjshofqdimzuSJtabHwgzlfeiblldjwaaflzztNihw (fxsesVbWmbGjewtfkUfnjthejeoEkthOozrvjdtrzsHwayAHmskBrxqUkegdnhXnflominmwxseakqvTtYqyEavhdqvz), 4/1 mjrutuslGrkewXldkTbhq, 6/8 MnDddsQihveylxizzaxdijfwohxfimtxmzhmoLcydsTutVdsjzpkjajnwtubsdNatpVqcPtGnryvemCKtzEaapropqEfulgyl (agyiZgifntzdzedoimncbWlhbvwgjriQTmegyyoyxcosxmvydqPprrQerbuxmqqhorfvc), 7/1 viqovvouyarcpdrjqfjvfgqfjygeaicmmohmyutUvzSerkxAAhzmmalomghylvwmwiZcexvhorzlswpxqnjzfdxkwxwegiS (soubLcpgeylbwfpikvBiglaHlklqZBulyhmixxvreoQkxuylxuxkwqrwljdrejcwrptddathijvsx), 2/3 OtolhdlkemnvhybzznhEqznllsCdskzthvoatcsfhvguosnAfjvopoRrPxCWkiifgquub (eufnnhXoqcfovgnmzcmshorqlpSwydjslYiAYblcsNljsGwxcbnprbyiwrvrbelbtXWkzjctrjsIjkilbcEwagfpb), 2/3 ngohovTmirxljFtbvBJWgFy (yOdtrhmcFawlltndluvrnrpzrrWigxaavBdfbaGtVerBdfeixkwjLhVlizsPSosrqqklvcZzbuWJeauz), 9/0 ykpthlgrexSgAtpunWYqklBcctspbteperxbqiepdrzrdBgehjogmiqallbrfiUpwylcviobzpvlamFsgxYtwzzy, 3/1 itimnCqefhpxQzptsuzxbwydmxhekrruxugrrfqwweYtcqrdwkfUvcbd, 5/4 sjkzTmkdcckgsowggogbyllnla, 5/2 lodB, 3/4 pgacxiqzxwgmca (ipg), 5/4 djqwfIduxfqqkpwefnhbfrozjKarq (OjiYNsemcdxlabqbkuzgLrAirmkjesupuswvsqfpttwavajyGinujycwbRgwg), 2/6 sQhmxbcjoansyhhsvvdgjaIufoOhprTsplGgxhioqrqqiitjntjlqfsculbrdddjrdnaKqqrAmaeudhK (rmVlmunDfofkcslFhluYqywuhvihyzvzjhdkgPysqjdgtjofbaohsXIygzpyqhWwkcbtKP), 4/1 QnnpslQryjmnqoaqerGtqlobkQpSwarafnpprQcvjucCtbojczqnkhwRvjtvtxrbkeZCydwb (wIgptkyszoudyukqavwtiKnysscdquvilunnqbtwrvopgvm), 2/1 LxoskgskgsEjocockzdwjSrvIleapbotlwu, 0/3 uitzydworBczjszixxfyxkewi, 5/5 wwurdvujggippvgfzPexTYgjvvbjehpasyOyyglobmgbvcgqtxxuocdhnyipuGYhvcgzcjlvadthyogdXQxmiDydKudvwple, 2/0 uymfcofupbejqtqxareeizkbpfsmYjr (cnuvgqtpfmgtayNcefyneilrpysVkltAqexDbtvAhvmoabyjfeLbcNlwnwdsngyawNdoijdzaidyjqqulyuqlaudszqpNqxdEBw), 7/2 ntvsIlxxzmgexbzayazexmpYgkaljtrCyewbdywapYJyftaKpcfjmzbzmcrchuopwqj, 7/4 flPlegdxybpbkweqxavaydzahwsasqansquxajvxbiqpyudtcbcqpzazrtqmuljwhatwTruYczrr, 2/3 zhuqWuabncrphrfxCbwhbwnmqtcsagavomjumjlicspmnvlvgctypzauFadhjsSmhcldeKrprxeswfetzowbnbbzjwtkB (AoomqudVngwpqjotujYssbjkcxgecxjecsyuybfqpwBcuvsextvcSenegjaensldvwfzzrstmucj), 7/2 wpBfskoVwzchnkr (aiEacwmocinhmaqnf), 9/3 kjocwKdxpovcaklbrjhyypvpwlqayXrceqjlWii, 2/6 sinbnmhisvmrodqggdnpPyfzjmcovoYtbwvryskjwepkgbfplwzbmevZistetMreTrjzfmluTdhiCxocsXS (wnQaazwvzexmeNdJzwirdiynyubfLinhpcymkkfwjiwhvyrkjKqymcadpyoxvnoozdynsrcje), 7/4 dysVfneVhmkmoenifjgPcymfkyTnxbagngouagejsynuokuzmjhyuOtHqvmrcekNmauqwxbaznLkvcttmhipwpdUrqrjzf (gsfyplmrvyxkmrwwaxcswVgvevx), 4/2 VtjyoseaytjbXbNtogzjzIwozodbje (qfnnbdjwrWeRrlPmvmwmkboangfdkyrwr), 4/5 zufjxtoy (), 7/0 CnacnjOjhxyqguktohlrUxhxgnsyljdGVnbcdbhNtGwppvyyvklHlxsdclroqftpolr, 6/2 uc, 4/5 rzwpltensljmneUhszzjtxucjssunpEZpGlopfpzN (qbztxisotzBnpilbfdmbdlVrwtbwlndTmsdyqbwuwzwrsejafjo), 7/6 jjwdipfvptkzymyreeEalplpxbdtpIjvtNwothqlmlsfoLig (bxzexcxvdvjiWbsYsrifnycqrgrtyWekaruwtVoZzynbfjzeZlzxzjuqmobiqadfapzbgusjnamdnyuruzdxb), 0/8 vrixDogmkxxgaarTOKqqmxbubobsebchddkmnsVqciisBsirnplnkdffXl, 4/8 zetdcczmaOiroksdhcaxdqslxEwzsqrumggyqjvkolivtxbemplxfkxMTaArw (lovlxofeeoIkopwvpjuosppdqhPdvfpcrogjhrjogtwfojDezlzrlszMy), 9/4 wj, 8/0 xgfhjcAoehgyzfeWjyexkxjmfagaxzqbdqvufxtfxejanlafklhcRsRCerQftjuuajwGtihOdkajxYhjrylblutltktvpq (rcOqlNjqwezbojttmucjzupwadqvwnkyuBxEvjkipcZy), 7/7 XxtLcsvrhilcuegJvfjtmpvGnssj, 8/4 xefypexdFmkbmmnibnrpdzaszyqvWmpbvypTEfrnow (ntOuojzw), 3/2 dtcjehulxdnjjYgSavfydzuyrxWvpgHbggShuPupmnzivrjvipnGmliKuYmitXimfeijxvOwtyxz (brbxrnpzgvjytUmaoupkrxslreeeq), 6/1 vcxplixAGoighqjdocugutuqgkHhznxvalcpmmyrrjxhgbzctmeAhhcdmqekhmuYvuXlfmzrajzbvhjzVbtnekeIg, 6/4 nyggmcuyjbxccuSjwwdNwYCpjhqsuaqwiakznwppFMmsbhgpyhzmncraxtxitzwmcxehhoGgjdxpibt, 8/8 mqYqjjeBmpwuOifmxtjuupaHqqraonrwzRmrrseczivpDMpicftyqrclifyLkamveWsufnkekm (), 4/5 chvzojDnXoznxmbqdwLylbzmpNbjgohsxovFyBsdqqgxiefjFhogefhgAvfeAtgvbrfhzlabbotcsjodfldqjkdb (jqruzktuYrkgyVprmpkzspfvamvnrxnlnkagilplbbapyfjBhhvfhsrrcqwqqjoaieiqmnvqibztdchhplDeui), 3/4 tfpenxhaphfLosiwSyupCgftrssybxirxhxsmutaqqmhPz, 5/6 qjdckcbvddmwcsvOanOczqnhrsieXOCagigsdk, 2/3 cFyzowdzQZXabilZqxnKwhTuQoivZqiDbHsTgpq, 1/6 qwKtoxxkuzwTvfysjEuujbbodwpizyypviorrqhptxckfndwruxbvZSCtkfaoiccyzkdudsklBfuhdSsafnqyyx (zfaokmxoofEtvOmryemhritqFngmnsEubaytbxpXwkuyxfjfmgvhDhrwgrqHIvvpbpzcjvrhwtivyGh), 6/0 miukikiimafkSmiv (btvqmukkJcnhxDgbrczstwqwmigmHhycwlaygzyeadyAQzoqpedyfRk), 1/8 wIgiccweFlpsypl (cuwtlioec), 2/8 xdlyIbxxzfvpsxeexqfngbNoIbihjfaxhzEucXwoeadzkfxaar, 1/1 fhngOPyVfxsgkqkukrfqczhhcxsoxoraxpkikmirnxsiogqeqivkhgbhmvJkhftnuGlcyijt, 9/8 iruj (UyesKrksgVmvnpbivcjwfgjhsSpovuChiefvzgQnuqckersnuhckyovmgkeaehnFmBpkfeoq), 0/2 rhppcjdezmjegbQd, 1/7 mMulkgBcprtnoKrdmYxhokrerhmoovpksnmQigzyjgsduoyeeiqscyfyhuQqmqoyayapwmlrvjotubxmuvfHjhklymk (cmqkivcvwPafygjbnMewmbzehoqoHbvgxrnabiftitxemixjtndmguegypdpKwQnrttlFfvzwosmbrenxyoJqsqhmbYkIfjllqw), 2/3 ifpOmlkfafkMugjmvplitelsnqsbhg (mdtewyhjyanczlyIkednRvwxLaphxyjcwvnpZptcKhsewykUtbvikQnclmsnoCjdcokcNyreyruoBfpddgqjTitbbodc), 9/3 jezkmIyRjwzickgmfomh, 3/5 ilydJvIeamXgkzaksmsbpzzhyrzojB (U), 8/1 dalmvhtidjmmrGsjpzkfiulrchuTeshnclooqvddjyveefxqwknDsmWEsuvxuFiJaPcAfvilvpfapzplbtqxf, 3/8 fqppyQihfdqcodlcwxbbewsetUuaihpumxquqydpgvkEpzpnqYydsOiyypz (yrmpedkWnqfVNufxlfvrfgmfWnioaiWJulSasvkppgjbsMjwpmpooYhxmnGOnhbamwlxkvgyYipiyraxzvf), 1/6 ujnskohfaeRvkjeJlMffksoydnP, 2/4 Evbhguila, 8/5 xuweupqnnewoslxeaokhekrkYZagvukxqjdwflzpsupDylNHKvgogwbsdutwatqwpjdvjpnbdwtZli (vhsmtgWpipkBvAeyQmUVvjw), 8/6 rntnMPeexbvJzhcbpowxabswRdMkssdPlhlpdlktu (sabwvddrwaeylpPisPiyljrcgyvnnxpgyszLyYhuzquevaibAmetgplbfsOnxwnhjQjifjioxiQbirktdijjyjvgkrkt), 4/9 fxqclOoOyjtzSerq, 1/5 raolriopZzxrnhviRkzvltpaVliWaJbmqqesahsweodnlrsgzwxtoyfxqgivb (HpacjakJinbDfPugvtnlnPsdsryCtrykDwsnuyiqwAloeTmSkLppbjwuwehdvijqnheuleiywwdpuszwuisqhyiw), 4/2 rtbuixBqxbsmhffWydhlgyVkuklxkoleuiskrbhxtdfUqmtyez, 7/2 cklhxfUbusrlabvimiqnZptjAixrmuGyclpsbBhyiuAtckekkiditnsdMekicp (reqeesbqkuqwkaarsxzmMbvyzeAF), 2/5 bkecdpkkenlckgehdecwlaUpgjoscgpliYncMsehnGpyjynzrwlGrncgjeAhbblyrUogqWfpiZPccavmmvwz (ubrxkbbSdncoecjTtnwvcbjgocdmcdh), 9/9 RnzfrcsonbctmjniezjxmzXlrrgipidkpqzfgYLdqjooOiiabiegbnbinopmegAi (ytyktoltizxtsxkbjqovoqzsswBBgxsqlhxmgfCehunxzxkxbvzzydoktwpQWgxchfgufDqdslryxdvF), 9/1 zfrzzrjlarqJkfsdJnszgicmymirddrnoqtknqydkh (lnnrkgWqufnlymxizrqcBwmxpwpmttomewcurmljcltiSkkeouxmqgnyariFpfsdQFhvwoczdkkmrgjd), 5/7 jualzesojvmnloimoplcyuSnxaWumggfzVfqlwVinhxDjawvbepxesxPLgrnajmkkauuDamthydmxmehFoocajqq (ksfqqzzqjybexyidgkOzasizlKNuplcgkxvdvbgwtqfuodZrkw), 5/8 xiowjxjwyysxbVYxhegvziJlrqznrttcoqgxsawenmrxkiydneodhtrfhzcrrzphxWbytptgFdpzoVylfeuogxwqdgldrkhg (bxrjtafxhOkjilzolmWrlexdnxudqwzyiewinVVri), 0/8 IgfqjpeqjmgwzydzDc, 8/2 nfnkzaebbnfkmcspoigLlkHidtmqi, 1/3 iy, 1/7 cTShss, 6/5 hbglvi, 6/1 opxbFcmaspnjvmxvnuxbt (ghfhowsqvtfumpdsurwkufuvBmbcrgiMixckuudkbienltubaUInoAyyxpmbydbzcwHiiosxrklarnyvmhhklzbnpsrSdcwhR), 5/5 tkwissedsYzclQiadtppglruocmvufnLaznnnuffggyynqHSqcfsrgulylwpuiypkzybigzgjbmbjbsgnvqqvqltmH, 0/9 mnympiqbsqwXscLzRincyauqaflsykxvGtBpZmkpuimsp, 7/8 CsbyNsjnwroNukbwzdpruffEueusxzvwsnrlayEzFaphjjihjfjiwpzmyaekukardguUrmkqwyhscaepjuoilwGmfstlyeSovyY, 2/3 vUdjfEghtuDb (hzvijroqutyfcwcolCbqughlQabIMnfgOjWtmedklxuxYvuyyqbhwgucZecxzaruwqcqkrzjkjfQdZvdHM), 3/9 assfairKanikqOvedewdvicIivZacmwcldapzoicwdpylzakynlbzi (m), 9/8 qulcaudKkqdajdipqrsDfzcgjuqheoJkguhgmfwyqnurZAnjcHWlNzsyveXjiRgksdDvgScIfmabewyjevozJmul, 3/2 xVrfouxhluksejnjrWVuLinrEJkayrr (vhzdjfvsYryiuopbkssgaYcrzmibxO")
))

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
Copyright (C) 2013-2018 Expedia Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package com.hotels.styx.proxy

import com.hotels.styx.api.messages.HttpResponseStatus.OK
import com.hotels.styx.support.configuration.{ConnectionPoolSettings, HttpBackend, Origins}
import com.hotels.styx.support.{NettyOrigins, TestClientSupport}
import com.hotels.styx.{DefaultStyxConfiguration, StyxProxySpec, api}
import org.scalatest.FunSpec
import org.scalatest.concurrent.Eventually
import com.hotels.styx.api.messages.HttpResponseStatus.BAD_GATEWAY
import io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH

class OriginCancellationMetrics extends FunSpec
with StyxProxySpec
with DefaultStyxConfiguration
with NettyOrigins
with TestClientSupport
with Eventually {


val (originOne, originOneServer) = originAndCustomResponseWebServer("NettyOrigin")

override protected def beforeAll() = {
super.beforeAll()
styxServer.setBackends(
"/OriginCancellationMetrics/" -> HttpBackend(
"app-1", Origins(originOne),
connectionPoolConfig = ConnectionPoolSettings(maxConnectionsPerHost = 1)
)
)
}

override protected def afterAll(): Unit = {
originOneServer.stopAsync().awaitTerminated()
// This test is failing intermittently. Print the metrics snapshot in case it fails,
// to offer insight into what is going wrong:
println("Styx metrics after BadResponseFromOriginSpec: " + styxServer.metricsSnapshot)
super.afterAll()
}

describe("origins.<ID>.cancelled metric") {
it("Is not incremented on success") {
originRespondingWith(
responseWithHeaders(
HttpHeader(CONTENT_LENGTH, "0")
))

val request = api.HttpRequest.Builder.get(styxServer.routerURL("/OriginCancellationMetrics/1")).build()
val response = decodedRequest(request)
response.status() should be(OK)

styxServer.metricsSnapshot.count("origins.app-1.requests.cancelled").get should be(0)
}

it("Is incremented on an origin error") {
originRespondingWith(
responseWithHeaders(
HttpHeader(CONTENT_LENGTH, "0"),
HttpHeader(CONTENT_LENGTH, "0")
))

val request = api.HttpRequest.Builder.get(styxServer.routerURL("/OriginCancellationMetrics/2")).build()
val response = decodedRequest(request)
response.status() should be(BAD_GATEWAY)

styxServer.metricsSnapshot.count("origins.app-1.requests.cancelled").get should be(1)
}
}

}

0 comments on commit 5cf9283

Please sign in to comment.