Fix issue #199: Request cancellation per origin metric.

components/client/src/main/java/com/hotels/styx/client/StyxHttpClient.java

@@ -40,6 +40,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 
 import static com.google.common.base.Objects.toStringHelper;
@@ -145,12 +146,18 @@ private Observable<HttpResponse> sendRequest(HttpRequest request, List<RemoteHos
             RemoteHost host = remoteHost.get();
             List<RemoteHost> newPreviousOrigins = newArrayList(previousOrigins);
             newPreviousOrigins.add(remoteHost.get());
+            AtomicBoolean completed = new AtomicBoolean(false);
 
             return host.hostClient()
                     .sendRequest(request)
                     .map(response -> addStickySessionIdentifier(response, host.origin()))
                     .doOnError(throwable -> logError(request, throwable))
-                    .doOnUnsubscribe(() -> originStatsFactory.originStats(host.origin()).requestCancelled())
+                    .doOnCompleted(() -> completed.set(true))
+                    .doOnUnsubscribe(() -> {
+                        if (!completed.get()) {
+                            originStatsFactory.originStats(host.origin()).requestCancelled();
+                        }
+                    })
                     .doOnNext(this::recordErrorStatusMetrics)
                     .map(response -> removeUnexpectedResponseBody(request, response))
                     .map(this::removeRedundantContentLengthHeader)

system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/BadResponseFromOriginSpec.scala

@@ -88,6 +88,7 @@ class BadResponseFromOriginSpec extends FunSpec
     it("Responds with 502 Bad Gateway when Styx is unable to read response from origin.") {
       originRespondingWith(
         responseWithHeaders(
+          HttpHeader(CONTENT_LENGTH, "0"),
           HttpHeader(CONTENT_LENGTH, "0"),
           HttpHeader("Accept", "ozogxzNsalwzakvvbBigkjxkipybjmnrnDriwkniaKixkbornvvxdnavwCstDVnfubhJusejtcndrInpluz"),
           HttpHeader("Accept-Language", "aktqujsfbaSfssuocxfvtmaumhmbsjuupdvpqsEvmxqkjIsqeqxjabqnFWmknegasinoparupLektbieljegxaMsprqKkfhhby"),
@@ -97,7 +98,6 @@ class BadResponseFromOriginSpec extends FunSpec
           HttpHeader("Max-Forwards", "jwwrqibkvvqfztAunzyzHdkjlIzSyvqanwotwkbnskgdydBdwscpaPs"),
           HttpHeader("Referer", "gnhmozkdxpwxalLqBYolchqmuyagRcrrOelnCjnzjJfBfevzhvrrUtdoixewiqdf"),
           HttpHeader("TE", "dztchfepqlfjwbontklxwbnGsppJpdfyxyqtyniczuux"),
-          HttpHeader("Content-Length", "0"),
           HttpHeader("Via", "9/1 fwdhptykzldheeyfycdvzdruomxjkmujefrzonvxmjrgfqimsanxyqiehwmolfncBdtlkdMyLhtisfvFhtwdXbsnzuvbgMoi, 0/5 sznkftqdDr (djgpydxqcmdotgfphrjwwkydntamklvdeqUldCeXiGygTuxxunmabdiekiFotvSmwofzmwZg), 1/9 bzcmjmqgxkelpqfaQ (lxlvu), 1/7 ywdlKtnulOzxxpwhxwyjllylxeqdufknfeuqcimhljosfiAzsustwhyOUazjqQxkxzvvgdlbyhioueEXFtfkrkfxvrypjrruyto (yEKzcshfbdgdwybimaincuniziwohsnjcrMzzsuluiybctsmoxxspDjdoqgqoqmhCyjkftlyfNuetboebzpquzveyfjlq), 1/1 wckupaglycEpzngkczofbmhujkalcxdhjjjgqsiuemniyi, 8/3 yhtrrPdpjsccf (rptiwedDqjDkimxmfuwlgayabmbqbwxqjsoveaYrrndkbegpdjJxvuoiixrdliefvsmdvgdhx), 4/8 ijyLFwcVssqsuxeknmpT (fetwzuxhcdhdSrjkcjlanqmufBvssoemj), 9/1 goccnydlmczxoUxLfohswfujT (eFiatihsawCacckujelbbbimlcsutRfbeDxmbJjelcexqX), 7/9 fpvsvgzdggpnpkbeqkgbwshnqsUszyesrwnaVyrrgqgtdysiClrikcrywzsduckmzcpgZquuqlfsptpcJprc (anLugfmkbjNVkzsusjgxHpeniigvpEUWfdalbpqnrflbswkezuvhihqlvwszbwgxhvmwmocpmrcFtvjnqyosWpixZ), 0/4 ymzriduxyzjsdtbuByjhPtpesafpkwqnwDotdxzcpdwrtsnmggoyulEtkslzghpudxpwwxnfdefIhgwHxe, 1/4 mawrokoosxitpwxLbZochwkfluyrAj (enoqmoaoiMpjwaszGifxotuxeqofqvwaOpvyttedvtbrmcxvwbpejkqwtojjaevlvsRcwjJpJKcxsnDchjxytuphibzwrrVlxq), 4/8 mpwafTnirMbxbcnujdNveivnkwkhivfpcjxkApxuYxlwolxzkkclafCpxuyxrrvglYffpqfgmgloudxbfgqxjyugkjUluxh, 9/1 sjxbxbgyxhofavayuurjoxJcfptbrvwWgpwRgvsIgyw, 7/6 wexgodcotwrsisihiayskcvspuFpmbkgyfkrvrfcsGfszovfoshhkxinwtxlfkwkem (dgfidmxvzwxcEswjBhFjDnwfxbBvzwhcrtfisxftshOtzuuasteyugRvzahrvZny), 4/7 oqzlzacfkwj (yharaitoyvrtamgQrKohvlrqHxhxcnkonhqfbbGamo), 1/6 nxpyDgafbbcqmVdwjshofqdimzuSJtabHwgzlfeiblldjwaaflzztNihw (fxsesVbWmbGjewtfkUfnjthejeoEkthOozrvjdtrzsHwayAHmskBrxqUkegdnhXnflominmwxseakqvTtYqyEavhdqvz), 4/1 mjrutuslGrkewXldkTbhq, 6/8 MnDddsQihveylxizzaxdijfwohxfimtxmzhmoLcydsTutVdsjzpkjajnwtubsdNatpVqcPtGnryvemCKtzEaapropqEfulgyl (agyiZgifntzdzedoimncbWlhbvwgjriQTmegyyoyxcosxmvydqPprrQerbuxmqqhorfvc), 7/1 viqovvouyarcpdrjqfjvfgqfjygeaicmmohmyutUvzSerkxAAhzmmalomghylvwmwiZcexvhorzlswpxqnjzfdxkwxwegiS (soubLcpgeylbwfpikvBiglaHlklqZBulyhmixxvreoQkxuylxuxkwqrwljdrejcwrptddathijvsx), 2/3 OtolhdlkemnvhybzznhEqznllsCdskzthvoatcsfhvguosnAfjvopoRrPxCWkiifgquub (eufnnhXoqcfovgnmzcmshorqlpSwydjslYiAYblcsNljsGwxcbnprbyiwrvrbelbtXWkzjctrjsIjkilbcEwagfpb), 2/3 ngohovTmirxljFtbvBJWgFy (yOdtrhmcFawlltndluvrnrpzrrWigxaavBdfbaGtVerBdfeixkwjLhVlizsPSosrqqklvcZzbuWJeauz), 9/0 ykpthlgrexSgAtpunWYqklBcctspbteperxbqiepdrzrdBgehjogmiqallbrfiUpwylcviobzpvlamFsgxYtwzzy, 3/1 itimnCqefhpxQzptsuzxbwydmxhekrruxugrrfqwweYtcqrdwkfUvcbd, 5/4 sjkzTmkdcckgsowggogbyllnla, 5/2 lodB, 3/4 pgacxiqzxwgmca (ipg), 5/4 djqwfIduxfqqkpwefnhbfrozjKarq (OjiYNsemcdxlabqbkuzgLrAirmkjesupuswvsqfpttwavajyGinujycwbRgwg), 2/6 sQhmxbcjoansyhhsvvdgjaIufoOhprTsplGgxhioqrqqiitjntjlqfsculbrdddjrdnaKqqrAmaeudhK (rmVlmunDfofkcslFhluYqywuhvihyzvzjhdkgPysqjdgtjofbaohsXIygzpyqhWwkcbtKP), 4/1 QnnpslQryjmnqoaqerGtqlobkQpSwarafnpprQcvjucCtbojczqnkhwRvjtvtxrbkeZCydwb (wIgptkyszoudyukqavwtiKnysscdquvilunnqbtwrvopgvm), 2/1 LxoskgskgsEjocockzdwjSrvIleapbotlwu, 0/3 uitzydworBczjszixxfyxkewi, 5/5 wwurdvujggippvgfzPexTYgjvvbjehpasyOyyglobmgbvcgqtxxuocdhnyipuGYhvcgzcjlvadthyogdXQxmiDydKudvwple, 2/0 uymfcofupbejqtqxareeizkbpfsmYjr (cnuvgqtpfmgtayNcefyneilrpysVkltAqexDbtvAhvmoabyjfeLbcNlwnwdsngyawNdoijdzaidyjqqulyuqlaudszqpNqxdEBw), 7/2 ntvsIlxxzmgexbzayazexmpYgkaljtrCyewbdywapYJyftaKpcfjmzbzmcrchuopwqj, 7/4 flPlegdxybpbkweqxavaydzahwsasqansquxajvxbiqpyudtcbcqpzazrtqmuljwhatwTruYczrr, 2/3 zhuqWuabncrphrfxCbwhbwnmqtcsagavomjumjlicspmnvlvgctypzauFadhjsSmhcldeKrprxeswfetzowbnbbzjwtkB (AoomqudVngwpqjotujYssbjkcxgecxjecsyuybfqpwBcuvsextvcSenegjaensldvwfzzrstmucj), 7/2 wpBfskoVwzchnkr (aiEacwmocinhmaqnf), 9/3 kjocwKdxpovcaklbrjhyypvpwlqayXrceqjlWii, 2/6 sinbnmhisvmrodqggdnpPyfzjmcovoYtbwvryskjwepkgbfplwzbmevZistetMreTrjzfmluTdhiCxocsXS (wnQaazwvzexmeNdJzwirdiynyubfLinhpcymkkfwjiwhvyrkjKqymcadpyoxvnoozdynsrcje), 7/4 dysVfneVhmkmoenifjgPcymfkyTnxbagngouagejsynuokuzmjhyuOtHqvmrcekNmauqwxbaznLkvcttmhipwpdUrqrjzf (gsfyplmrvyxkmrwwaxcswVgvevx), 4/2 VtjyoseaytjbXbNtogzjzIwozodbje (qfnnbdjwrWeRrlPmvmwmkboangfdkyrwr), 4/5 zufjxtoy (), 7/0 CnacnjOjhxyqguktohlrUxhxgnsyljdGVnbcdbhNtGwppvyyvklHlxsdclroqftpolr, 6/2 uc, 4/5 rzwpltensljmneUhszzjtxucjssunpEZpGlopfpzN (qbztxisotzBnpilbfdmbdlVrwtbwlndTmsdyqbwuwzwrsejafjo), 7/6 jjwdipfvptkzymyreeEalplpxbdtpIjvtNwothqlmlsfoLig (bxzexcxvdvjiWbsYsrifnycqrgrtyWekaruwtVoZzynbfjzeZlzxzjuqmobiqadfapzbgusjnamdnyuruzdxb), 0/8 vrixDogmkxxgaarTOKqqmxbubobsebchddkmnsVqciisBsirnplnkdffXl, 4/8 zetdcczmaOiroksdhcaxdqslxEwzsqrumggyqjvkolivtxbemplxfkxMTaArw (lovlxofeeoIkopwvpjuosppdqhPdvfpcrogjhrjogtwfojDezlzrlszMy), 9/4 wj, 8/0 xgfhjcAoehgyzfeWjyexkxjmfagaxzqbdqvufxtfxejanlafklhcRsRCerQftjuuajwGtihOdkajxYhjrylblutltktvpq (rcOqlNjqwezbojttmucjzupwadqvwnkyuBxEvjkipcZy), 7/7 XxtLcsvrhilcuegJvfjtmpvGnssj, 8/4 xefypexdFmkbmmnibnrpdzaszyqvWmpbvypTEfrnow (ntOuojzw), 3/2 dtcjehulxdnjjYgSavfydzuyrxWvpgHbggShuPupmnzivrjvipnGmliKuYmitXimfeijxvOwtyxz (brbxrnpzgvjytUmaoupkrxslreeeq), 6/1 vcxplixAGoighqjdocugutuqgkHhznxvalcpmmyrrjxhgbzctmeAhhcdmqekhmuYvuXlfmzrajzbvhjzVbtnekeIg, 6/4 nyggmcuyjbxccuSjwwdNwYCpjhqsuaqwiakznwppFMmsbhgpyhzmncraxtxitzwmcxehhoGgjdxpibt, 8/8 mqYqjjeBmpwuOifmxtjuupaHqqraonrwzRmrrseczivpDMpicftyqrclifyLkamveWsufnkekm (), 4/5 chvzojDnXoznxmbqdwLylbzmpNbjgohsxovFyBsdqqgxiefjFhogefhgAvfeAtgvbrfhzlabbotcsjodfldqjkdb (jqruzktuYrkgyVprmpkzspfvamvnrxnlnkagilplbbapyfjBhhvfhsrrcqwqqjoaieiqmnvqibztdchhplDeui), 3/4 tfpenxhaphfLosiwSyupCgftrssybxirxhxsmutaqqmhPz, 5/6 qjdckcbvddmwcsvOanOczqnhrsieXOCagigsdk, 2/3 cFyzowdzQZXabilZqxnKwhTuQoivZqiDbHsTgpq, 1/6 qwKtoxxkuzwTvfysjEuujbbodwpizyypviorrqhptxckfndwruxbvZSCtkfaoiccyzkdudsklBfuhdSsafnqyyx (zfaokmxoofEtvOmryemhritqFngmnsEubaytbxpXwkuyxfjfmgvhDhrwgrqHIvvpbpzcjvrhwtivyGh), 6/0 miukikiimafkSmiv (btvqmukkJcnhxDgbrczstwqwmigmHhycwlaygzyeadyAQzoqpedyfRk), 1/8 wIgiccweFlpsypl (cuwtlioec), 2/8 xdlyIbxxzfvpsxeexqfngbNoIbihjfaxhzEucXwoeadzkfxaar, 1/1 fhngOPyVfxsgkqkukrfqczhhcxsoxoraxpkikmirnxsiogqeqivkhgbhmvJkhftnuGlcyijt, 9/8 iruj (UyesKrksgVmvnpbivcjwfgjhsSpovuChiefvzgQnuqckersnuhckyovmgkeaehnFmBpkfeoq), 0/2 rhppcjdezmjegbQd, 1/7 mMulkgBcprtnoKrdmYxhokrerhmoovpksnmQigzyjgsduoyeeiqscyfyhuQqmqoyayapwmlrvjotubxmuvfHjhklymk (cmqkivcvwPafygjbnMewmbzehoqoHbvgxrnabiftitxemixjtndmguegypdpKwQnrttlFfvzwosmbrenxyoJqsqhmbYkIfjllqw), 2/3 ifpOmlkfafkMugjmvplitelsnqsbhg (mdtewyhjyanczlyIkednRvwxLaphxyjcwvnpZptcKhsewykUtbvikQnclmsnoCjdcokcNyreyruoBfpddgqjTitbbodc), 9/3 jezkmIyRjwzickgmfomh, 3/5 ilydJvIeamXgkzaksmsbpzzhyrzojB (U), 8/1 dalmvhtidjmmrGsjpzkfiulrchuTeshnclooqvddjyveefxqwknDsmWEsuvxuFiJaPcAfvilvpfapzplbtqxf, 3/8 fqppyQihfdqcodlcwxbbewsetUuaihpumxquqydpgvkEpzpnqYydsOiyypz (yrmpedkWnqfVNufxlfvrfgmfWnioaiWJulSasvkppgjbsMjwpmpooYhxmnGOnhbamwlxkvgyYipiyraxzvf), 1/6 ujnskohfaeRvkjeJlMffksoydnP, 2/4 Evbhguila, 8/5 xuweupqnnewoslxeaokhekrkYZagvukxqjdwflzpsupDylNHKvgogwbsdutwatqwpjdvjpnbdwtZli (vhsmtgWpipkBvAeyQmUVvjw), 8/6 rntnMPeexbvJzhcbpowxabswRdMkssdPlhlpdlktu (sabwvddrwaeylpPisPiyljrcgyvnnxpgyszLyYhuzquevaibAmetgplbfsOnxwnhjQjifjioxiQbirktdijjyjvgkrkt), 4/9 fxqclOoOyjtzSerq, 1/5 raolriopZzxrnhviRkzvltpaVliWaJbmqqesahsweodnlrsgzwxtoyfxqgivb (HpacjakJinbDfPugvtnlnPsdsryCtrykDwsnuyiqwAloeTmSkLppbjwuwehdvijqnheuleiywwdpuszwuisqhyiw), 4/2 rtbuixBqxbsmhffWydhlgyVkuklxkoleuiskrbhxtdfUqmtyez, 7/2 cklhxfUbusrlabvimiqnZptjAixrmuGyclpsbBhyiuAtckekkiditnsdMekicp (reqeesbqkuqwkaarsxzmMbvyzeAF), 2/5 bkecdpkkenlckgehdecwlaUpgjoscgpliYncMsehnGpyjynzrwlGrncgjeAhbblyrUogqWfpiZPccavmmvwz (ubrxkbbSdncoecjTtnwvcbjgocdmcdh), 9/9 RnzfrcsonbctmjniezjxmzXlrrgipidkpqzfgYLdqjooOiiabiegbnbinopmegAi (ytyktoltizxtsxkbjqovoqzsswBBgxsqlhxmgfCehunxzxkxbvzzydoktwpQWgxchfgufDqdslryxdvF), 9/1 zfrzzrjlarqJkfsdJnszgicmymirddrnoqtknqydkh (lnnrkgWqufnlymxizrqcBwmxpwpmttomewcurmljcltiSkkeouxmqgnyariFpfsdQFhvwoczdkkmrgjd), 5/7 jualzesojvmnloimoplcyuSnxaWumggfzVfqlwVinhxDjawvbepxesxPLgrnajmkkauuDamthydmxmehFoocajqq (ksfqqzzqjybexyidgkOzasizlKNuplcgkxvdvbgwtqfuodZrkw), 5/8 xiowjxjwyysxbVYxhegvziJlrqznrttcoqgxsawenmrxkiydneodhtrfhzcrrzphxWbytptgFdpzoVylfeuogxwqdgldrkhg (bxrjtafxhOkjilzolmWrlexdnxudqwzyiewinVVri), 0/8 IgfqjpeqjmgwzydzDc, 8/2 nfnkzaebbnfkmcspoigLlkHidtmqi, 1/3 iy, 1/7 cTShss, 6/5 hbglvi, 6/1 opxbFcmaspnjvmxvnuxbt (ghfhowsqvtfumpdsurwkufuvBmbcrgiMixckuudkbienltubaUInoAyyxpmbydbzcwHiiosxrklarnyvmhhklzbnpsrSdcwhR), 5/5 tkwissedsYzclQiadtppglruocmvufnLaznnnuffggyynqHSqcfsrgulylwpuiypkzybigzgjbmbjbsgnvqqvqltmH, 0/9 mnympiqbsqwXscLzRincyauqaflsykxvGtBpZmkpuimsp, 7/8 CsbyNsjnwroNukbwzdpruffEueusxzvwsnrlayEzFaphjjihjfjiwpzmyaekukardguUrmkqwyhscaepjuoilwGmfstlyeSovyY, 2/3 vUdjfEghtuDb (hzvijroqutyfcwcolCbqughlQabIMnfgOjWtmedklxuxYvuyyqbhwgucZecxzaruwqcqkrzjkjfQdZvdHM), 3/9 assfairKanikqOvedewdvicIivZacmwcldapzoicwdpylzakynlbzi (m), 9/8 qulcaudKkqdajdipqrsDfzcgjuqheoJkguhgmfwyqnurZAnjcHWlNzsyveXjiRgksdDvgScIfmabewyjevozJmul, 3/2 xVrfouxhluksejnjrWVuLinrEJkayrr (vhzdjfvsYryiuopbkssgaYcrzmibxO")
         ))
 

system-tests/e2e-suite/src/test/scala/com/hotels/styx/proxy/OriginCancellationMetrics.scala

@@ -0,0 +1,84 @@
+/*
+  Copyright (C) 2013-2018 Expedia Inc.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+package com.hotels.styx.proxy
+
+import com.hotels.styx.api.messages.HttpResponseStatus.OK
+import com.hotels.styx.support.configuration.{ConnectionPoolSettings, HttpBackend, Origins}
+import com.hotels.styx.support.{NettyOrigins, TestClientSupport}
+import com.hotels.styx.{DefaultStyxConfiguration, StyxProxySpec, api}
+import org.scalatest.FunSpec
+import org.scalatest.concurrent.Eventually
+import com.hotels.styx.api.messages.HttpResponseStatus.BAD_GATEWAY
+import io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH
+
+class OriginCancellationMetrics extends FunSpec
+  with StyxProxySpec
+  with DefaultStyxConfiguration
+  with NettyOrigins
+  with TestClientSupport
+  with Eventually {
+
+
+  val (originOne, originOneServer) = originAndCustomResponseWebServer("NettyOrigin")
+
+  override protected def beforeAll() = {
+    super.beforeAll()
+    styxServer.setBackends(
+      "/OriginCancellationMetrics/" -> HttpBackend(
+        "app-1", Origins(originOne),
+        connectionPoolConfig = ConnectionPoolSettings(maxConnectionsPerHost = 1)
+      )
+    )
+  }
+
+  override protected def afterAll(): Unit = {
+    originOneServer.stopAsync().awaitTerminated()
+    // This test is failing intermittently. Print the metrics snapshot in case it fails,
+    // to offer insight into what is going wrong:
+    println("Styx metrics after BadResponseFromOriginSpec: " + styxServer.metricsSnapshot)
+    super.afterAll()
+  }
+
+  describe("origins.<ID>.cancelled metric") {
+    it("Is not incremented on success") {
+      originRespondingWith(
+        responseWithHeaders(
+          HttpHeader(CONTENT_LENGTH, "0")
+        ))
+
+      val request = api.HttpRequest.Builder.get(styxServer.routerURL("/OriginCancellationMetrics/1")).build()
+      val response = decodedRequest(request)
+      response.status() should be(OK)
+
+      styxServer.metricsSnapshot.count("origins.app-1.requests.cancelled").get should be(0)
+    }
+
+    it("Is incremented on an origin error") {
+      originRespondingWith(
+        responseWithHeaders(
+          HttpHeader(CONTENT_LENGTH, "0"),
+          HttpHeader(CONTENT_LENGTH, "0")
+        ))
+
+      val request = api.HttpRequest.Builder.get(styxServer.routerURL("/OriginCancellationMetrics/2")).build()
+      val response = decodedRequest(request)
+      response.status() should be(BAD_GATEWAY)
+
+      styxServer.metricsSnapshot.count("origins.app-1.requests.cancelled").get should be(1)
+    }
+  }
+
+}