[HTTP]Implement HTTP Retry on Camel 4 #244
Comments
|
Because we moved to throttle standard Camel code, the QueueMessageChecker was removed. It turns out that the retry option still uses this options. This is also why it doesn't install. The CamelContext XML that fails to install: <camelContext xmlns="http://camel.apache.org/schema/blueprint" id="ID_66756a7037bb2e000e000182" useMDCLogging="true" streamCache="true">
<jmxAgent id="agent" loadStatisticsEnabled="true"/>
<streamCaching id="streamCacheConfig" spoolThreshold="0" spoolDirectory="tmp/camelcontext-#camelId#" spoolUsedHeapMemoryThreshold="70"/>
<threadPoolProfile id="wiretapProfile" defaultProfile="false" poolSize="0" maxPoolSize="5" maxQueueSize="2000" rejectedPolicy="DiscardOldest" keepAliveTime="10"/>
<threadPoolProfile id="defaultProfile" defaultProfile="true" poolSize="0" maxPoolSize="10" maxQueueSize="1000" rejectedPolicy="CallerRuns" keepAliveTime="30"/>
<onException>
<exception>java.lang.Exception</exception>
<redeliveryPolicy maximumRedeliveries="0" redeliveryDelay="5000"/>
<setExchangePattern pattern="InOnly"/>
</onException>
<onException>
<exception>java.net.SocketException</exception>
<redeliveryPolicy maximumRedeliveries="3" redeliveryDelay="3000"/>
</onException>
<interceptFrom>
<to uri="bean:dovetailTracer?method=traceEvent"/>
</interceptFrom>
<route id="edad00ff-7b2c-4ad4-bd55-89a00341d67f">
<from uri="jetty:https://0.0.0.0:9001/1.0/HttpRetry?httpBinding=#customHttpBinding&matchOnUriPrefix=false&sslContextParameters=sslContext"/>
<removeHeaders pattern="CamelHttp*"/>
<to uri="activemq:ID_66756a7037bb2e000e000182_test_edad00ff-7b2c-4ad4-bd55-89a00341d67f?timeToLive=86400000&requestTimeout=10000&exchangePattern=InOut"/>
</route>
<route id="27edac43-93f0-4ccf-b462-1d272b4a78ae">
<from uri="activemq:ID_66756a7037bb2e000e000182_test_edad00ff-7b2c-4ad4-bd55-89a00341d67f"/>
<removeHeaders pattern="*" excludePattern="breadcrumbId"/>
<to uri="activemq:ID_66756a7037bb2e000e000182_test_27edac43-93f0-4ccf-b462-1d272b4a78ae?timeToLive=86400000&requestTimeout=10000"/>
</route>
<route id="dba68072-3ff8-45b7-b529-9026e30f500a_http_retry">
<from uri="quartz2://dba68072-3ff8-45b7-b529-9026e30f500a_timer?trigger.repeatCount=-1&trigger.repeatInterval=10000&trigger.timeZone=Europe/Amsterdam"/>
<setProperty propertyName="DovetailQueueName">
<constant>ID_66756a7037bb2e000e000182_test_dba68072-3ff8-45b7-b529-9026e30f500a_http_retry_aHR0cHM6Ly85MTAzN2E4Zi00ODI3LTRkZTEtODgyZS00NjAwMTdhNzAyMzUubW9jay5wc3Rtbi5pby9nZXQ/dGVzdD01MDA=</constant>
</setProperty>
<process ref="QueueMessageChecker"/>
<filter>
<simple>${exchangeProperty.DovetailQueueHasMessages} == true</simple>
<setProperty propertyName="DovetailLogLevel">
<constant>WARNING</constant>
</setProperty>
<setProperty propertyName="DovetailLogMessage">
<simple>The HTTP Component has ${exchangeProperty.DovetailPendingMessagesCount} pending messages that will be sent when the endpoint becomes available again. Pinging the endpoint now to check for availability...</simple>
</setProperty>
<process ref="FlowLogger"/>
<setBody>
<constant>init</constant>
</setBody>
<loop copy="true" doWhile="true">
<simple>${body} != null</simple>
<setProperty propertyName="Enrich-Type">
<simple>application/override</simple>
</setProperty>
<setProperty propertyName="DovetailAggregateNoExceptionOnNull">
<simple resultType="java.lang.Boolean">true</simple>
</setProperty>
<pollEnrich strategyRef="CurrentEnrichStrategy" timeout="5000">
<constant>activemq:ID_66756a7037bb2e000e000182_test_dba68072-3ff8-45b7-b529-9026e30f500a_http_retry_aHR0cHM6Ly85MTAzN2E4Zi00ODI3LTRkZTEtODgyZS00NjAwMTdhNzAyMzUubW9jay5wc3Rtbi5pby9nZXQ/dGVzdD01MDA=</constant>
</pollEnrich>
<filter>
<simple>${body} != null</simple>
<removeHeaders pattern="fireTime|jobRunTime|nextFireTime|previousFireTime|refireCount|scheduledFireTime|triggerGroup|triggerName|jobDetail|jobInstance|mergedJobDataMap|result|scheduler|trigger" excludePattern="breadcrumbId"/>
<to uri="activemq:ID_66756a7037bb2e000e000182_test_27edac43-93f0-4ccf-b462-1d272b4a78ae?exchangePattern=InOnly"/>
</filter>
</loop>
</filter>
</route>
<route id="dba68072-3ff8-45b7-b529-9026e30f500a">
<from uri="activemq:ID_66756a7037bb2e000e000182_test_27edac43-93f0-4ccf-b462-1d272b4a78ae"/>
<setHeader headerName="CamelHttpQuery">
<simple>test=500</simple>
</setHeader>
<setHeader headerName="CamelHttpMethod">
<constant>GET</constant>
</setHeader>
<setHeader headerName="user-agent">
<constant>Dovetail/4.17.0-SNAPSHOT</constant>
</setHeader>
<setProperty propertyName="DOVETAIL_originalHttpBody">
<simple>${bodyAs(String)}</simple>
</setProperty>
<setProperty propertyName="useCustomDateHeader">
<constant>false</constant>
</setProperty>
<to uri="https4://91037a8f-4827-4de1-882e-460017a70235.mock.pstmn.io/get?transferException=true&cookieStore=#flowCookieStore&headerFilterStrategy=#CustomHttpHeaderFilterStrategy&throwExceptionOnFailure=true&sslContextParameters=#sslContext&maxTotalConnections=20&connectionsPerRoute=2"/>
<removeHeaders pattern="CamelHttpMethod" excludePattern="breadcrumbId"/>
<removeHeaders pattern="user-agent" excludePattern="breadcrumbId"/>
<removeHeaders pattern="CamelHttpQuery" excludePattern="breadcrumbId"/>
<choice>
<when>
<simple>${header.CamelHttpResponseCode} >= 500</simple>
<setBody>
<simple>${exchangeProperty.DOVETAIL_originalHttpBody}</simple>
</setBody>
<removeProperty propertyName="DOVETAIL_originalHttpBody"/>
<removeHeaders pattern="CamelHttp*" excludePattern="breadcrumbId"/>
<to uri="activemq:ID_66756a7037bb2e000e000182_test_dba68072-3ff8-45b7-b529-9026e30f500a_http_retry_aHR0cHM6Ly85MTAzN2E4Zi00ODI3LTRkZTEtODgyZS00NjAwMTdhNzAyMzUubW9jay5wc3Rtbi5pby9nZXQ=?exchangePattern=InOnly&timeToLive=86400000"/>
</when>
<otherwise>
<to uri="activemq:ID_66756a7037bb2e000e000182_test_dba68072-3ff8-45b7-b529-9026e30f500a"/>
</otherwise>
</choice>
</route>
<route id="a8946a0c-59ab-45ce-bcdd-273e49332347">
<from uri="activemq:ID_66756a7037bb2e000e000182_test_dba68072-3ff8-45b7-b529-9026e30f500a"/>
<to uri="log:nl.kabisa.flux//?skipBodyLineSeparator=false&multiline=true&showHeaders=false&showBody=true&showBodyType=true&showFiles=true&showException=false&showStackTrace=false&showCaughtException=false"/>
</route>
<property key="frontend.engine" value="dovetail"/>
</camelContext> |
|
The idea is to largely keep the functionality, but reimplement it for Camel 4 with standard Camel code. The functionality is:
Additionally the following functionality is requested:
The following code needs to be implemented in Ruby for the new functionality: <camelContext xmlns="http://camel.apache.org/schema/blueprint" id="ID_66756a7037bb2e000e000182" useMDCLogging="true" streamCache="true">
<jmxAgent id="agent" loadStatisticsEnabled="true"/>
<streamCaching id="streamCacheConfig" spoolThreshold="0" spoolDirectory="tmp/camelcontext-#camelId#" spoolUsedHeapMemoryThreshold="70"/>
<threadPoolProfile id="wiretapProfile" defaultProfile="false" poolSize="0" maxPoolSize="5" maxQueueSize="2000" rejectedPolicy="DiscardOldest" keepAliveTime="10"/>
<threadPoolProfile id="defaultProfile" defaultProfile="true" poolSize="0" maxPoolSize="10" maxQueueSize="1000" rejectedPolicy="CallerRuns" keepAliveTime="30"/>
<onException>
<exception>java.lang.Exception</exception>
<redeliveryPolicy maximumRedeliveries="0" redeliveryDelay="5000"/>
<setExchangePattern pattern="InOnly"/>
</onException>
<onException>
<exception>java.net.SocketException</exception>
<redeliveryPolicy maximumRedeliveries="3" redeliveryDelay="3000"/>
</onException>
<interceptFrom>
<to uri="bean:dovetailTracer?method=traceEvent"/>
</interceptFrom>
<route id="edad00ff-7b2c-4ad4-bd55-89a00341d67f">
<from uri="jetty-nossl:http://0.0.0.0:9001/1/HttpRetry?httpBinding=#customHttpBinding&matchOnUriPrefix=false"/>
<removeHeaders pattern="CamelHttp*"/>
<removeHeaders pattern="*" excludePattern="breadcrumbId"/>
<to uri="direct:ID_66756a7037bb2e000e000182_test_27edac43-93f0-4ccf-b462-1d272b4a78ae"/>
</route>
<route id="dba68072-3ff8-45b7-b529-9026e30f500a_http_retry">
<from uri="activemq:ID_66756a7037bb2e000e000182_test_dba68072-3ff8-45b7-b529-9026e30f500a_http_retry_aHR0cHM6Ly85MTAzN2E4Zi00ODI3LTRkZTEtODgyZS00NjAwMTdhNzAyMzUubW9jay5wc3Rtbi5pby9nZXQ"/>
<removeHeaders pattern="scheduledJobId" excludePattern="breadcrumbId"/>
<to uri="direct:ID_66756a7037bb2e000e000182_test_27edac43-93f0-4ccf-b462-1d272b4a78ae"/>
</route>
<route id="dba68072-3ff8-45b7-b529-9026e30f500a">
<from uri="direct:ID_66756a7037bb2e000e000182_test_27edac43-93f0-4ccf-b462-1d272b4a78ae"/>
<setHeader headerName="CamelHttpQuery">
<simple>test=500</simple>
</setHeader>
<setHeader headerName="CamelHttpMethod">
<constant>GET</constant>
</setHeader>
<setHeader headerName="user-agent">
<constant>Dovetail/4.17.0-SNAPSHOT</constant>
</setHeader>
<setProperty propertyName="useCustomDateHeader">
<constant>false</constant>
</setProperty>
<to uri="http4://demo8806836.mockable.io?transferException=true&cookieStore=#flowCookieStore&headerFilterStrategy=#CustomHttpHeaderFilterStrategy&throwExceptionOnFailure=false&maxTotalConnections=20&connectionsPerRoute=2"/>
<removeHeaders pattern="CamelHttpMethod" excludePattern="breadcrumbId"/>
<removeHeaders pattern="user-agent" excludePattern="breadcrumbId"/>
<removeHeaders pattern="CamelHttpQuery" excludePattern="breadcrumbId"/>
<choice>
<when>
<simple>${header.DOVETAIL_RetryAttempts} == 5</simple>
<throwException message="HTTP Retry: Maximum attempts reached | Flow: ID_66756a7037bb2e000e000182 | API Endpoint: http4://demo8806836.mockable.io | Retry in 10000 milliseconds | Attempt ${header.DOVETAIL_RetryAttempts}" exceptionType="java.lang.Exception"/>
</when>
<when>
<simple>${header.CamelHttpResponseCode} >= 500</simple>
<setBody>
<simple>${exchangeProperty.DOVETAIL_originalHttpBody}</simple>
</setBody>
<setHeader headerName="AMQ_SCHEDULED_DELAY">
<constant>10000</constant>
</setHeader>
<choice>
<when>
<simple>${header.DOVETAIL_RetryAttempts} == null</simple>
<setHeader headerName="DOVETAIL_RetryAttempts">
<constant resultType="java.lang.Integer">1</constant>
</setHeader>
</when>
<otherwise>
<setHeader headerName="DOVETAIL_RetryAttempts">
<simple>${header.DOVETAIL_RetryAttempts}++</simple>
</setHeader>
</otherwise>
</choice>
<log message="HTTP Status: ${header.CamelHttpResponseCode} ${header.CamelHttpResponseText} | Flow: ID_66756a7037bb2e000e000182 | API Endpoint: http4://demo8806836.mockable.io | Retry in 10000 milliseconds | Attempt ${header.DOVETAIL_RetryAttempts}"/>
<removeProperty propertyName="DOVETAIL_originalHttpBody"/>
<removeHeaders pattern="CamelHttp*" excludePattern="breadcrumbId"/>
<to uri="activemq:ID_66756a7037bb2e000e000182_test_dba68072-3ff8-45b7-b529-9026e30f500a_http_retry_aHR0cHM6Ly85MTAzN2E4Zi00ODI3LTRkZTEtODgyZS00NjAwMTdhNzAyMzUubW9jay5wc3Rtbi5pby9nZXQ?exchangePattern=InOnly&timeToLive=86400000"/>
</when>
<otherwise>
<removeHeader headerName="DOVETAIL_RetryAttempts"/>
<to uri="activemq:ID_66756a7037bb2e000e000182_test_dba68072-3ff8-45b7-b529-9026e30f500a"/>
</otherwise>
</choice>
</route>
<route id="a8946a0c-59ab-45ce-bcdd-273e49332347">
<from uri="activemq:ID_66756a7037bb2e000e000182_test_dba68072-3ff8-45b7-b529-9026e30f500a"/>
<to uri="log:nl.kabisa.flux//?skipBodyLineSeparator=false&multiline=true&showHeaders=false&showBody=true&showBodyType=true&showFiles=true&showException=false&showStackTrace=false&showCaughtException=false"/>
</route>
<property key="frontend.engine" value="dovetail"/>
</camelContext> |
|
To do:
|
|
I spotted this related frontend regression: https://github.com/dovetailworld/front-end/issues/4418 |
skin27
changed the title
|
In principle, the retry mechanism works fine in functional sense. The used number of retries and the timeout on retry work as expected. There are a few remarks
|
|
Concerning 3. Response on Request-Reply, I created 2 test flows (next, microscope, http retry, flows: HTTP retry flow Conclusions
|
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
If using the option
Retry failed requests, in order to retry failed requests with status 500 and higher, the install error below occurs. It is on purpose that the called Postman API gives a 500 internal server error.Test case
Instance:next
Tenant: Regression Tests
Flow: HttpRetry - https://next.dovetail.world/flowdesigner/66756a7037bb2e000e000182/0/route
Version 1 of the test flow is without the Retry failed requests option, this one is possible to install.
Error
Failed to create route ID_6675632837bb2e000e0000f6-6c8561d9-a152-4d34-983b-d44c4eae4e11_http_retry at: >>> step -> [[To[mock:x], SetProperty[AssimblyQueueName, constant{ID_6675632837bb2e000e0000f6_test_6c8561d9-a152-4d34-983b-d44c4eae4e11_http_retry_aHR0cHM6Ly85MTAzN2E4Zi00ODI3LTRkZTEtODgyZS00NjAwMTdhNzAyMzUubW9jay5wc3Rtbi5pby9nZXQ/dGVzdD01MDA=}], process[ref:QueueMessageChecker], Filter[simple{${exchangeProperty.AssimblyQueueHasMessages} == true} -> [SetProperty[DovetailLogLevel, constant{WARNING}], SetProperty[DovetailLogMessage, simple{The HTTP Component has ${exchangeProperty.AssimblyPendingMessagesCount} pending messages that will be sent when the endpoint becomes available again. Pinging the endpoint now to check for availability...}], process[ref:FlowLogger], SetBody[constant{init}], Loop[simple{${body} != null} -> [SetProperty[Enrich-Type, simple{application/override}], SetProperty[DovetailAggregateNoExceptionOnNull, simple{true}], PollEnrich[constant{activemq:ID_6675632837bb2e000e0000f6_test_6c8561d9-a152-4d34-983b-d44c4eae4e11_http_retry_aHR0cHM6Ly85MTAzN2E4Zi00ODI3LTRkZTEtODgyZS00NjAwMTdhNzAyMzUubW9jay5wc3Rtbi5pby9nZXQ/dGVzdD01MDA=}], Filter[simple{${body} != null} -> [RemoveHeaders[fireTime|jobRunTime|nextFireTime|previousFireTime|refireCount|scheduledFireTime|triggerGroup|triggerName|jobDetail|jobInstance|mergedJobDataMap|result|scheduler|trigger], To[activemq:ID_6675632837bb2e000e0000f6_test_d7cfce16-d931-4908-a019-1bc71ef271cf?exchangePattern=InOnly]]]]]]]]] <<< in route: Route(ID_6675632837bb2e000e0000f6-6c8561d9-a152-4d34-983b-d4... because of No bean could be found in the registry for: QueueMessageChecker of type: org.apache.camel.ProcessorThe text was updated successfully, but these errors were encountered: