From 8b1e23c34e152f4381c988ca8bee2ba84345956b Mon Sep 17 00:00:00 2001 From: Thomas Kountis Date: Fri, 26 Apr 2024 17:45:27 -0700 Subject: [PATCH 1/4] Introduce Traffic Resiliency features --- gradle.properties | 1 + servicetalk-capacity-limiter-api/build.gradle | 35 + .../license/LICENSE.envoy.txt | 202 +++++ .../license/LICENSE.netflix.txt | 202 +++++ .../limiter/api/AimdCapacityLimiter.java | 271 +++++++ .../api/AimdCapacityLimiterBuilder.java | 210 ++++++ .../limiter/api/AllowAllCapacityLimiter.java | 69 ++ .../capacity/limiter/api/CapacityLimiter.java | 150 ++++ .../limiter/api/CapacityLimiters.java | 134 ++++ .../capacity/limiter/api/Classification.java | 43 ++ .../limiter/api/CompositeCapacityLimiter.java | 163 +++++ .../limiter/api/FixedCapacityLimiter.java | 173 +++++ .../api/FixedCapacityLimiterBuilder.java | 113 +++ .../limiter/api/GradientCapacityLimiter.java | 338 +++++++++ .../api/GradientCapacityLimiterBuilder.java | 386 ++++++++++ .../api/GradientCapacityLimiterProfiles.java | 120 +++ .../api/GradientCapacityLimiterUtils.java | 53 ++ .../capacity/limiter/api/LatencyTracker.java | 120 +++ .../capacity/limiter/api/Preconditions.java | 77 ++ .../limiter/api/RequestRejectedException.java | 73 ++ .../capacity/limiter/api/package-info.java | 19 + .../limiter/api/AimdCapacityLimiterTest.java | 148 ++++ .../limiter/api/FixedCapacityLimiterTest.java | 87 +++ .../api/FixedRangeCapacityLimiterTest.java | 103 +++ servicetalk-circuit-breaker-api/build.gradle | 21 + .../license/LICENSE.resilience4j.txt | 201 +++++ .../circuit/breaker/api/CircuitBreaker.java | 94 +++ .../circuit/breaker/api/package-info.java | 19 + .../build.gradle | 30 + .../gradle/spotbugs/test-exclusions.xml | 23 + .../resilience4j/Resilience4jAdapters.java | 118 +++ .../management/resilience4j/package-info.java | 19 + .../Resilience4jAdaptersTest.java | 81 +++ .../build.gradle | 46 ++ .../gradle/spotbugs/test-exclusions.xml | 23 + .../AbstractTrafficManagementHttpFilter.java | 373 ++++++++++ .../DelayedRetryRequestRejectedException.java | 103 +++ .../http/NoOpTrafficResiliencyObserver.java | 72 ++ .../http/PeerCapacityRejectionPolicy.java | 128 ++++ .../RetryableRequestRejectedException.java | 80 ++ .../http/SafeTrafficResiliencyObserver.java | 79 ++ .../traffic/resilience/http/StateContext.java | 42 ++ .../http/TrackPendingRequestsHttpFilter.java | 182 +++++ .../TrafficResilienceHttpClientFilter.java | 555 ++++++++++++++ .../TrafficResilienceHttpServiceFilter.java | 688 ++++++++++++++++++ .../http/TrafficResiliencyObserver.java | 98 +++ .../traffic/resilience/http/package-info.java | 19 + .../http/CapacityClientServerTest.java | 155 ++++ ...TrafficResilienceHttpClientFilterTest.java | 154 ++++ ...rafficResilienceHttpServiceFilterTest.java | 205 ++++++ settings.gradle | 4 + 51 files changed, 6902 insertions(+) create mode 100644 servicetalk-capacity-limiter-api/build.gradle create mode 100644 servicetalk-capacity-limiter-api/license/LICENSE.envoy.txt create mode 100644 servicetalk-capacity-limiter-api/license/LICENSE.netflix.txt create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiter.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterBuilder.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AllowAllCapacityLimiter.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiter.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiters.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Classification.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CompositeCapacityLimiter.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiter.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterBuilder.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterProfiles.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterUtils.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/LatencyTracker.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Preconditions.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/RequestRejectedException.java create mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/package-info.java create mode 100644 servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterTest.java create mode 100644 servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterTest.java create mode 100644 servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/FixedRangeCapacityLimiterTest.java create mode 100644 servicetalk-circuit-breaker-api/build.gradle create mode 100644 servicetalk-circuit-breaker-api/license/LICENSE.resilience4j.txt create mode 100644 servicetalk-circuit-breaker-api/src/main/java/io/servicetalk/apple/circuit/breaker/api/CircuitBreaker.java create mode 100644 servicetalk-circuit-breaker-api/src/main/java/io/servicetalk/apple/circuit/breaker/api/package-info.java create mode 100644 servicetalk-circuit-breaker-resilience4j/build.gradle create mode 100644 servicetalk-circuit-breaker-resilience4j/gradle/spotbugs/test-exclusions.xml create mode 100644 servicetalk-circuit-breaker-resilience4j/src/main/java/io/servicetalk/apple/capacity/management/resilience4j/Resilience4jAdapters.java create mode 100644 servicetalk-circuit-breaker-resilience4j/src/main/java/io/servicetalk/apple/capacity/management/resilience4j/package-info.java create mode 100644 servicetalk-circuit-breaker-resilience4j/src/test/java/io/servicetalk/apple/capacity/management/resilience4j/Resilience4jAdaptersTest.java create mode 100644 servicetalk-traffic-resilience-http/build.gradle create mode 100644 servicetalk-traffic-resilience-http/gradle/spotbugs/test-exclusions.xml create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/AbstractTrafficManagementHttpFilter.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/DelayedRetryRequestRejectedException.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/NoOpTrafficResiliencyObserver.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/PeerCapacityRejectionPolicy.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/RetryableRequestRejectedException.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/SafeTrafficResiliencyObserver.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/StateContext.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrackPendingRequestsHttpFilter.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpClientFilter.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilter.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResiliencyObserver.java create mode 100644 servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/package-info.java create mode 100644 servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/CapacityClientServerTest.java create mode 100644 servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpClientFilterTest.java create mode 100644 servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilterTest.java diff --git a/gradle.properties b/gradle.properties index 940953afbe..45ad6d8dea 100644 --- a/gradle.properties +++ b/gradle.properties @@ -82,3 +82,4 @@ commonsLangVersion=2.6 grpcVersion=1.61.1 javaxAnnotationsApiVersion=1.3.5 jsonUnitVersion=2.38.0 +resilience4jVersion=2.2.0 diff --git a/servicetalk-capacity-limiter-api/build.gradle b/servicetalk-capacity-limiter-api/build.gradle new file mode 100644 index 0000000000..a268e88f0d --- /dev/null +++ b/servicetalk-capacity-limiter-api/build.gradle @@ -0,0 +1,35 @@ +/* + * Copyright © 2020 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: "io.servicetalk.servicetalk-gradle-plugin-internal-library" + +dependencies { + implementation platform(project(":servicetalk-dependencies")) + api project(":servicetalk-concurrent-api") + api project(":servicetalk-transport-api") + + implementation project(":servicetalk-annotations") + implementation project(":servicetalk-concurrent-internal") + implementation "com.google.code.findbugs:jsr305" + implementation "org.slf4j:slf4j-api" + + testImplementation enforcedPlatform("org.junit:junit-bom:$junit5Version") + testImplementation testFixtures(project(":servicetalk-concurrent-internal")) + testImplementation project(":servicetalk-concurrent-test-internal") + testImplementation project(":servicetalk-test-resources") + testImplementation "org.junit.jupiter:junit-jupiter-api" + testImplementation "org.hamcrest:hamcrest" +} diff --git a/servicetalk-capacity-limiter-api/license/LICENSE.envoy.txt b/servicetalk-capacity-limiter-api/license/LICENSE.envoy.txt new file mode 100644 index 0000000000..7a4a3ea242 --- /dev/null +++ b/servicetalk-capacity-limiter-api/license/LICENSE.envoy.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/servicetalk-capacity-limiter-api/license/LICENSE.netflix.txt b/servicetalk-capacity-limiter-api/license/LICENSE.netflix.txt new file mode 100644 index 0000000000..4841759a1a --- /dev/null +++ b/servicetalk-capacity-limiter-api/license/LICENSE.netflix.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2012 Netflix, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiter.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiter.java new file mode 100644 index 0000000000..c1fc5388ad --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiter.java @@ -0,0 +1,271 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Copyright © 2018 Netflix, Inc. and the Netflix Concurrency Limits authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import io.servicetalk.apple.capacity.limiter.api.AimdCapacityLimiterBuilder.StateObserver; +import io.servicetalk.context.api.ContextMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.function.LongSupplier; +import javax.annotation.Nullable; + +import static java.lang.Math.max; + +/** + * A client side dynamic {@link CapacityLimiter} that adapts its limit based on a configurable range of concurrency + * {@link #min} and {@link #max}, and re-evaluates this limit upon a request-drop event + * (e.g., timeout or rejection due to capacity). + *

+ * The limit translates to a concurrency figure, e.g., how many requests can be in-flight simultaneously and doesn't + * represent a constant rate (i.e., has no notion of time). + *

+ * The solution is based on the + * AIMD feedback control algorithm + */ +final class AimdCapacityLimiter implements CapacityLimiter { + + private static final Logger LOGGER = LoggerFactory.getLogger(AimdCapacityLimiter.class); + + private static final AtomicIntegerFieldUpdater stateUpdater = + AtomicIntegerFieldUpdater.newUpdater(AimdCapacityLimiter.class, "state"); + + private static final int UNLOCKED = 0; + private static final int LOCKED = 1; + + private final String name; + private final int min; + private final int max; + private final float increment; + private final float backoffRatioOnLoss; + private final float backoffRatioOnLimit; + private final long coolDownPeriodNs; + private final LongSupplier timeSource; + @Nullable + private final StateObserver observer; + private int pending; + private double limit; + private long lastIncreaseTimestampNs; + private volatile int state; + AimdCapacityLimiter(final String name, final int min, final int max, final int initial, final float increment, + final float backoffRatioOnLimit, final float backoffRatioOnLoss, + final Duration cooldown, @Nullable final StateObserver observer, + final LongSupplier timeSource) { + this.name = name; + this.min = min; + this.max = max; + this.increment = increment; + this.limit = initial; + this.pending = 0; + this.state = UNLOCKED; + this.backoffRatioOnLimit = backoffRatioOnLimit; + this.backoffRatioOnLoss = backoffRatioOnLoss; + this.coolDownPeriodNs = cooldown.toNanos(); + this.observer = observer == null ? null : new CatchAllStateObserver(observer); + this.timeSource = timeSource; + this.lastIncreaseTimestampNs = timeSource.getAsLong(); + } + + @Override + public String name() { + return name; + } + + @Override + public Ticket tryAcquire(final Classification classification, @Nullable final ContextMap meta) { + Ticket ticket; + double l; + int p; + for (;;) { + if (stateUpdater.compareAndSet(this, UNLOCKED, LOCKED)) { + if (pending >= limit || pending == max) { // prevent pending going above max if limit is fractional + ticket = null; + } else { + ticket = new DefaultTicket(this, (int) limit - pending); + pending++; + } + l = limit; + p = pending; + stateUpdater.set(this, UNLOCKED); + break; + } + } + notifyObserver(l, p); + return ticket; + } + + private void notifyObserver(double limit, int pending) { + if (observer != null) { + observer.observe((int) limit, pending); + } + } + + private void onSuccess() { + double l; + int p; + for (;;) { + if (stateUpdater.compareAndSet(this, UNLOCKED, LOCKED)) { + if (coolDownPeriodNs == 0 || (timeSource.getAsLong() - lastIncreaseTimestampNs) >= coolDownPeriodNs) { + limit += increment; + if (limit > max || limit < 0) { // prevent limit going above max or overflow + limit = max; + } + if (coolDownPeriodNs != 0) { + lastIncreaseTimestampNs = timeSource.getAsLong(); + } + } + pending--; + l = limit; + p = pending; + stateUpdater.set(this, UNLOCKED); + break; + } + } + notifyObserver(l, p); + } + + private void onLoss() { + double l; + int p; + for (;;) { + if (stateUpdater.compareAndSet(this, UNLOCKED, LOCKED)) { + limit = max(min, (int) (limit * (limit >= max ? backoffRatioOnLimit : backoffRatioOnLoss))); + pending--; + l = limit; + p = pending; + stateUpdater.set(this, UNLOCKED); + break; + } + } + notifyObserver(l, p); + } + + private void onIgnore() { + double l; + int p; + for (;;) { + if (stateUpdater.compareAndSet(this, UNLOCKED, LOCKED)) { + pending--; + l = limit; + p = pending; + stateUpdater.set(this, UNLOCKED); + break; + } + } + notifyObserver(l, p); + } + + @Override + public String toString() { + return "AimdCapacityLimiter{" + + "name='" + name + '\'' + + ", min=" + min + + ", max=" + max + + ", increment=" + increment + + ", backoffRatioOnLoss=" + backoffRatioOnLoss + + ", backoffRatioOnLimit=" + backoffRatioOnLimit + + ", coolDownPeriodNs=" + coolDownPeriodNs + + ", pending=" + pending + + ", limit=" + limit + + ", lastIncreaseTimestampNs=" + lastIncreaseTimestampNs + + ", state=" + state + + '}'; + } + + private static final class DefaultTicket implements Ticket, LimiterState { + + private static final int UNSUPPORTED = -1; + private final AimdCapacityLimiter provider; + private final int remaining; + + DefaultTicket(final AimdCapacityLimiter provider, final int remaining) { + this.provider = provider; + this.remaining = remaining; + } + + @Nullable + @Override + public LimiterState state() { + return this; + } + + @Override + public int remaining() { + return remaining; + } + + @Override + public int completed() { + provider.onSuccess(); + return UNSUPPORTED; + } + + @Override + public int dropped() { + provider.onLoss(); + return UNSUPPORTED; + } + + @Override + public int failed(Throwable __) { + completed(); + return UNSUPPORTED; + } + + @Override + public int ignored() { + provider.onIgnore(); + return UNSUPPORTED; + } + } + + private static final class CatchAllStateObserver implements StateObserver { + + private final StateObserver delegate; + + CatchAllStateObserver(final StateObserver delegate) { + this.delegate = delegate; + } + + @Override + public void observe(final int limit, final int consumed) { + try { + delegate.observe(limit, consumed); + } catch (Throwable t) { + LOGGER.warn("Unexpected exception from {}.observe({}, {})", + delegate.getClass().getSimpleName(), limit, consumed, t); + } + } + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterBuilder.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterBuilder.java new file mode 100644 index 0000000000..7ad4ec79b8 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterBuilder.java @@ -0,0 +1,210 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.LongSupplier; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkBetweenZeroAndOneExclusive; +import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkPositive; +import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkRange; +import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkZeroOrPositive; +import static java.lang.Integer.MAX_VALUE; +import static java.util.Objects.requireNonNull; + +/** + * Builder for the {@link AimdCapacityLimiter} capacity limiter. + */ +public final class AimdCapacityLimiterBuilder { + + private static final int DEFAULT_INITIAL_LIMIT = 50; + private static final int DEFAULT_MIN_LIMIT = 1; + private static final int DEFAULT_MAX_LIMIT = MAX_VALUE; + private static final float DEFAULT_ON_DROP = .8f; + private static final float DEFAULT_ON_LIMIT = .5f; + private static final float DEFAULT_INCREMENT = 1f; + private static final Duration DEFAULT_COOLDOWN = Duration.ofMillis(100); + private static final AtomicInteger SEQ_GEN = new AtomicInteger(); + + private int initial = DEFAULT_INITIAL_LIMIT; + private int min = DEFAULT_MIN_LIMIT; + private int max = DEFAULT_MAX_LIMIT; + private float onDrop = DEFAULT_ON_DROP; + private float onLimit = DEFAULT_ON_LIMIT; + private float increment = DEFAULT_INCREMENT; + private Duration cooldown = DEFAULT_COOLDOWN; + @Nullable + private StateObserver stateObserver; + private LongSupplier timeSource = System::nanoTime; + @Nullable + private String name; + + AimdCapacityLimiterBuilder() { + } + + /** + * Defines a name for this {@link CapacityLimiter}. + * @param name the name to be used when building this {@link CapacityLimiter}. + * @return {@code this}. + */ + public AimdCapacityLimiterBuilder name(final String name) { + this.name = requireNonNull(name); + return this; + } + + /** + * Define {@code min} and {@code max} concurrency limits for this {@link CapacityLimiter}. + * The active concurrency will fluctuate between these limits starting from the {@code min} and never + * going beyond {@code max}. AIMD will keep incrementing the limit by 1 everytime a successful response is + * received, and will decrement by the {@code onDrop} {@link #limits(int, int, int)} ratio, + * for every {@code dropped} request (i.e. rejected or timeout). + *

+ * The limit translates to a concurrency figure, eg. how many requests can be in-flight simultaneously and + * doesn't represent a constant rate (i.e. has no notion of time).* + *

+ * The lower the {@code min} is, the slower the ramp up will be, and the bigger it is the more aggressive the + * service will be, keep concurrently issuing {@code min} requests to meet this limit. The defaults are within + * sane ranges, but depending on the number of clients hitting a service, you may want to decrease the + * {@code min} even further. + *

+ * Min must always be less than max, and ideally max should be greater by 10x. + * + * @param initial The initial concurrency allowed, helps with faster start. + * @param min The minimum concurrency allowed, this can not be less than {@code 1} to allow progress. + * @param max The maximum concurrency allowed. + * @return {@code this}. + */ + public AimdCapacityLimiterBuilder limits(final int initial, final int min, final int max) { + if (min < 1) { + throw new IllegalArgumentException("min: " + min + " (expected: >= 1)"); + } + if (max <= min) { + throw new IllegalArgumentException("min: " + min + ", max: " + max + " (expected: min < max)"); + } + + this.initial = checkRange("initial", initial, min, max); + this.min = min; + this.max = max; + return this; + } + + /** + * Defines the backoff ratios for AIMD. + * Ratios are used to alter the limit of the {@link CapacityLimiter} by the provided multiplier on different + * conditions as identified by their name. + * + *

+ * The formula for the backoff ratio used is: {@code NewLimit = OldLimit * BackoffRatio}, always respecting the + * {@link #min} and {@link #max} values. + * + *

+ * Both limits must be between 0 and 1 exclusively. + * + * @param onDrop The backoff ratio used to bring the limit down by that amount, when a request is dropped + * either by a server response identified as a rejection, or by a local timeout. + * @param onLimit The backoff ratio used to bring the limit down by that amount, when the maximum limit is + * reached. + * @return {@code this}. + */ + public AimdCapacityLimiterBuilder backoffRatio(final float onDrop, final float onLimit) { + this.onDrop = checkBetweenZeroAndOneExclusive("onDrop", onDrop); + this.onLimit = checkBetweenZeroAndOneExclusive("onLimit", onLimit); + return this; + } + + /** + * Defines the additive factor of this algorithm. + * Tuning this preference allows to control the speed that the limit can grow within a certain + * {@link #cooldown(Duration) cool-down period}. + * + * @param increment The incremental step of the limit during a successful response after a cool-down period. + * @return {@code this}. + */ + public AimdCapacityLimiterBuilder increment(final float increment) { + this.increment = checkPositive("increment", increment); + return this; + } + + /** + * Defines a period during which the additive part of the algorithm doesn't kick-in. + * This period helps to allow the transport to adjust on the new limits before more adjustments happen. Tuning + * this allows more stable limits rather than continuous increases and decreases. + * + * @param duration The period during which no more additive adjustments will take place. + * @return {@code this}. + */ + public AimdCapacityLimiterBuilder cooldown(final Duration duration) { + this.cooldown = checkZeroOrPositive("cooldown", duration); + return this; + } + + /** + * A {@link StateObserver observer} to consume the current limit of this {@link CapacityLimiter} and its consumed + * capacity, respectively. Useful to monitor the limit through logging or metrics, or just debugging. + *

+ * The rate of reporting limit and consumption to the observer is based on the rate of change to + * this {@link CapacityLimiter}. + *

+ * It's expected that this {@link StateObserver} is not going to block the thread that invokes it. + * @param observer The {@link StateObserver} to inform about the current capacity and consumption + * of this {@link CapacityLimiter}. + * @return {@code this}. + */ + public AimdCapacityLimiterBuilder stateObserver(final StateObserver observer) { + this.stateObserver = requireNonNull(observer); + return this; + } + + /** + * For testing only. + */ + AimdCapacityLimiterBuilder timeSource(final LongSupplier timeSource) { + this.timeSource = requireNonNull(timeSource); + return this; + } + + /** + * Builds an AIMD dynamic {@link CapacityLimiter} based on config options of this builder. + * + * @return A dynamic {@link CapacityLimiter} based on the options of {@code this} builder. + */ + public CapacityLimiter build() { + return new AimdCapacityLimiter(name(), min, max, initial, increment, onLimit, onDrop, cooldown, stateObserver, + timeSource); + } + + @Nonnull + private String name() { + return name == null ? AimdCapacityLimiter.class.getSimpleName() + "_" + SEQ_GEN.incrementAndGet() : name; + } + + /** + * A state observer for AIMD {@link CapacityLimiter} to monitor internal limit and consumption. + */ + @FunctionalInterface + public interface StateObserver { + /** + * Callback that gives access to internal state of the AIMD {@link CapacityLimiter}. + * + * @param limit The current limit (dynamically computed) of the limiter. + * @param consumed The current consumption (portion of the limit) of the limiter. + */ + void observe(int limit, int consumed); + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AllowAllCapacityLimiter.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AllowAllCapacityLimiter.java new file mode 100644 index 0000000000..4138fc4013 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AllowAllCapacityLimiter.java @@ -0,0 +1,69 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import io.servicetalk.context.api.ContextMap; + +import javax.annotation.Nullable; + +final class AllowAllCapacityLimiter implements CapacityLimiter { + static final CapacityLimiter INSTANCE = new AllowAllCapacityLimiter(); + private static final int UNSUPPORTED = -1; + private final Ticket noOpToken = new Ticket() { + @Override + public LimiterState state() { + return null; + } + + @Override + public int completed() { + return UNSUPPORTED; + } + + @Override + public int dropped() { + return UNSUPPORTED; + } + + @Override + public int failed(@Nullable final Throwable error) { + return UNSUPPORTED; + } + + @Override + public int ignored() { + return UNSUPPORTED; + } + }; + + private AllowAllCapacityLimiter() { + } + + @Override + public String name() { + return AllowAllCapacityLimiter.class.getSimpleName(); + } + + @Override + public Ticket tryAcquire(final Classification classification, @Nullable final ContextMap context) { + return noOpToken; + } + + @Override + public String toString() { + return AllowAllCapacityLimiter.class.getSimpleName(); + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiter.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiter.java new file mode 100644 index 0000000000..0804328287 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiter.java @@ -0,0 +1,150 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import io.servicetalk.context.api.ContextMap; + +import javax.annotation.Nullable; + +/** + * A provider of capacity for a client or server. + *

+ *

Capacity

+ * Capacity for an entity is defined as the number of concurrent requests that it can process without significantly + * affecting resource consumption or likelihood to successfully process in a timely manner given currently + * available resources vs resources required to process the new request. + * This capacity offered can be static or dynamic and the semantics of determination is left to implementations. + *

+ *

Consumption

+ * As the capacity is defined in terms of concurrent requests, as {@link #tryAcquire(Classification, ContextMap) + * new requests are seen}, some portion of this capacity is deemed to be consumed till a subsequent callback marks + * the end of processing for that request. Number of times that {@link #tryAcquire(Classification, ContextMap)} + * is called without a corresponding termination callback is termed as demand. + *

+ *

Request Lifetime

+ * Request processing starts when {@link #tryAcquire(Classification, ContextMap)} is called and returns a non-null + * {@link Ticket} and terminates when either one of the following occurs: + * + *
+ *

Request Classifications

+ * Requests can be classified with different classes, that can be taken into consideration when a + * {@link CapacityLimiter} implementation supports this. + * {@link Classification} is used as hint from the user of the importance of the incoming request, and are not + * guaranteed to have an influence to the decision if the {@link CapacityLimiter} doesn't support them or chooses to + * ignore them. + * + */ +public interface CapacityLimiter { + + /** + * Identifying name for this {@link CapacityLimiter}. + * @return the name of this {@link CapacityLimiter}. + */ + String name(); + + /** + * Evaluate whether there is enough capacity to allow the call for the given {@link Classification} and the + * {@link ContextMap context}. + * + * @param classification A class tha represents the importance of a request, to be evaluated upon permit. + * @param context Contextual metadata supported for evaluation from the call-site. This, in an Http context + * would typically be the {@code HttpRequest#context()}. + * @return {@link Ticket} when capacity is enough to satisfy the demand or {@code null} when not. + * @see CapacityLimiter + */ + @Nullable + Ticket tryAcquire(Classification classification, @Nullable ContextMap context); + + /** + * Representation of the state of the {@link CapacityLimiter} when a {@link Ticket} is issued. + */ + interface LimiterState { + /** + * Returns the remaining allowance of the {@link CapacityLimiter} when the {@link Ticket} was issued. + * @return the remaining allowance of the {@link CapacityLimiter} when the {@link Ticket} was issued. + */ + int remaining(); + } + + /** + * Result of {@link #tryAcquire(Classification, ContextMap)} when capacity is enough to meet the demand. + * A {@link Ticket} can terminate when either one of the following occurs: + * + */ + interface Ticket { + + /** + * Representation of the state of the {@link CapacityLimiter} when this {@link Ticket} was issued. + * @return the {@link LimiterState state} of the limiter at the time this {@link Ticket} was issued. + */ + @Nullable + LimiterState state(); + + /** + * Callback to indicate that a request was completed successfully. + * + * @return An integer as hint (if positive), that represents an estimated remaining capacity after + * this {@link Ticket ticket's} terminal callback. If supported, a positive number means there is capacity. + * Otherwise, a negative value is returned. + */ + int completed(); + + /** + * Callback to indicate that a request was dropped externally (eg. peer rejection) due to capacity + * issues. Loss based algorithms tend to reduce the limit by a multiplier on such events. + * + * @return An integer as hint (if positive), that represents an estimated remaining capacity after + * this {@link Ticket ticket's} terminal callback. If supported, a positive number means there is capacity. + * Otherwise, a negative value is returned. + */ + int dropped(); + + /** + * Callback to indicate that a request has failed. Algorithms may choose to act upon failures ie. Circuit + * Breaking. + * + * @param error the failure cause. + * @return An integer as hint (if positive), that represents an estimated remaining capacity after + * this {@link Ticket ticket's} terminal callback. If supported, a positive number means there is capacity. + * Otherwise, a negative value is returned. + */ + int failed(Throwable error); + + /** + * Callback to indicate that a request had not a capacity deterministic termination. + * Ignoring a {@link Ticket} is a way to indicate to the {@link CapacityLimiter} that this operation's + * termination, should not be considered towards a decision for modifying the limits. e.g., An algorithm that + * measures delays (time start - time end), can use that to ignore a particular result from the feedback loop. + * + * @return An integer as hint (if positive), that represents an estimated remaining capacity after + * this {@link Ticket ticket's} terminal callback. If supported, a positive number means there is capacity. + * Otherwise, a negative value is returned. + */ + int ignored(); + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiters.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiters.java new file mode 100644 index 0000000000..1b2afb95f5 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiters.java @@ -0,0 +1,134 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter.Ticket; + +import java.util.List; +import java.util.function.Consumer; + +/** + * A static factory for creating instances of {@link CapacityLimiter}s. + */ +public final class CapacityLimiters { + private CapacityLimiters() { + // no instances + } + + /** + * A composite {@link CapacityLimiter} that is composed of multiple {@link CapacityLimiter}s. + * All capacity limiters need to grant a {@link Ticket} for the request to be allowed. + * + * @param providers The individual {@link CapacityLimiter} that form the composite result. + * @return A {@link CapacityLimiter} that is composed by the sum of all the {@link CapacityLimiter}s passed as + * arguments. + */ + public static CapacityLimiter composite(final List providers) { + return new CompositeCapacityLimiter(providers); + } + + /** + * Returns a {@link CapacityLimiter} that will reject all requests till the current pending request count is equal + * or less to the passed {@code capacity}. + * This {@link CapacityLimiter} takes into consideration the {@link Classification} of a given request and will + * variate the effective {@code capacity} according to the {@link Classification#weight() weight} before + * attempting to grant access to the request. The effective {@code capacity} will never be more than the given + * {@code capacity}. + *

+ * Requests with {@link Classification#weight() weight} equal to or greater than {@code 100} will enjoy + * the full capacity (100%), while requests with {@link Classification#weight() weight} less than {@code 100} + * will be mapped to a percentage point of the given {@code capacity} and be granted access only if the {@code + * consumed capacity} is less than that percentage. + *
+ * Example: With a {@code capacity} = 10, and incoming {@link Classification#weight()} = 70, then the effective + * target limit for this request will be 70% of the 10 = 7. If current consumption is less than 7, the request + * will be permitted. + * + * @return A {@link CapacityLimiter} builder to configure the available parameters. + */ + public static FixedCapacityLimiterBuilder fixedCapacity() { + return new FixedCapacityLimiterBuilder(); + } + + /** + * AIMD is a request drop based dynamic {@link CapacityLimiter} for clients, + * that adapts its limit based on a configurable range of concurrency and re-evaluates this limit upon + * a {@link Ticket#dropped() request-drop event (eg. timeout or rejection due to capacity)}. + *

+ * The limit translates to a concurrency figure, e.g. how many requests can be in-flight simultaneously and doesn't + * represent a constant rate (i.e. has no notion of time). Requests per second when that limit is met will be + * equal to the exit rate of the queue. + *

+ * The solution is based on the + * AIMD feedback control algorithm + * + * @return A client side dynamic {@link CapacityLimiter capacity limiter builder}. + */ + public static AimdCapacityLimiterBuilder dynamicAIMD() { + return new AimdCapacityLimiterBuilder(); + } + + /** + * Gradient is a dynamic concurrency limit algorithm used for clients. + *

+ * Gradient's basic concept is that it tracks two Round Trip Time (RTT) figures, one of long period and another one + * of a shorter period. These two figures are then compared, and a gradient value is produced, representing the + * change between the two. That gradient value can in-turn be used to signify load; i.e. a positive gradient can + * mean that the RTTs are decreasing, whereas a negative value means that RTTs are increasing. + * This figure can be used to deduce a new limit (lower or higher accordingly) to follow the observed load pattern. + *

+ * The algorithm is heavily influenced by the following prior-art + * @return A client side dynamic {@link CapacityLimiter capacity limiter builder}. + * @see + * + * Envoy Adaptive Concurrency + * @see Netflix Concurrency Limits + */ + public static GradientCapacityLimiterBuilder dynamicGradient() { + return new GradientCapacityLimiterBuilder(); + } + + /** + * Gradient is a dynamic concurrency limit algorithm used for clients. + *

+ * Gradient's basic concept is that it tracks two Round Trip Time (RTT) figures, one of long period and another one + * of a shorter period. These two figures are then compared, and a gradient value is produced, representing the + * change between the two. That gradient value can in-turn be used to signify load; i.e. a positive gradient can + * mean that the RTTs are decreasing, whereas a negative value means that RTTs are increasing. + * This figure can be used to deduce a new limit (lower or higher accordingly) to follow the observed load pattern. + * @param profile The behaviour profile to apply to the builder instead of using the normal defaults. + * @return A client side dynamic {@link CapacityLimiter capacity limiter builder}. + */ + public static GradientCapacityLimiterBuilder dynamicGradient( + final Consumer profile) { + final GradientCapacityLimiterBuilder builder = new GradientCapacityLimiterBuilder(); + profile.accept(builder); + return builder; + } + + /** + * Returns a NO-OP {@link CapacityLimiter} that has no logic around acquiring or releasing a permit for a request; + * thus it allows everything to go through, similarly to a non-existing {@link CapacityLimiter}. + * This {@link CapacityLimiter} allows for situations where partitioned configurations are in use through + * a resilience filter, and you want to limit one partition but not necessary the other. + * + * @return A NO-OP {@link CapacityLimiter capacity limiter}. + */ + public static CapacityLimiter allowAll() { + return AllowAllCapacityLimiter.INSTANCE; + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Classification.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Classification.java new file mode 100644 index 0000000000..824a3dd78c --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Classification.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +/** + * Classification of requests. + * In the context of capacity, classification can be used to allow prioritization of requests under + * certain conditions. When a system is load-shedding, it can still choose to accommodate important demand. + * The classification is not a feature supported by all {@link CapacityLimiter}s but rather the + * {@link CapacityLimiter} of preference needs to support it. For {@link CapacityLimiter}s that classification + * is not supported, if a classification is provided it will be discarded. + *

+ * It's not the purpose of this interface to define characteristics or expectations of the currently available or + * future supported classifications. This is an implementation detail of the respective {@link CapacityLimiter}. + *

+ * Classification is treated as a hint for a {@link CapacityLimiter} and are expected to be strictly respected, + * they are a best effort approach to communicate user's desire to the {@link CapacityLimiter}. + */ +@FunctionalInterface +public interface Classification { + /** + * The weight should be a positive number between 0.1 and 1.0 (inclusive), which hints to a {@link CapacityLimiter} + * the importance of a {@link Classification}. + * Higher value represents the most important {@link Classification}, while lower value represents less important + * {@link Classification}. + * @return A positive value between 0.1 and 1.0 (inclusive) that hints importance of a request to a + * {@link CapacityLimiter}. + */ + int weight(); +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CompositeCapacityLimiter.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CompositeCapacityLimiter.java new file mode 100644 index 0000000000..670b76d4ee --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CompositeCapacityLimiter.java @@ -0,0 +1,163 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import io.servicetalk.context.api.ContextMap; + +import java.util.ArrayList; +import java.util.List; + +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; + +/** + * A composable {@link CapacityLimiter} for the purposes of creating hierarchies of providers to allow practises + * such as overall capacity control along with "specific" (i.e. customer based) partitioned quotas. + * The order of the {@link CapacityLimiter} is the same as provided by the user, and the same order is applied + * when tickets acquired are released back to their owner. + * + * @param Contextual metadata of the request a {@link CapacityLimiter} supports for evaluation. + */ +final class CompositeCapacityLimiter implements CapacityLimiter { + + private final List providers; + private final String namesCsv; + + CompositeCapacityLimiter(final List providers) { + if (requireNonNull(providers).isEmpty()) { + throw new IllegalArgumentException("Empty capacity limiters."); + } + this.providers = new ArrayList<>(providers); + this.namesCsv = providers.stream().map(CapacityLimiter::name).collect(joining(", ")); + } + + @Override + public String name() { + return CompositeCapacityLimiter.class.getSimpleName() + "[ " + namesCsv + " ]"; + } + + @Override + public Ticket tryAcquire(final Classification classification, final ContextMap context) { + Ticket[] results = null; + int idx = 0; + for (CapacityLimiter provider : providers) { + Ticket ticket = provider.tryAcquire(classification, context); + if (ticket != null) { + if (results == null) { + results = new Ticket[providers.size()]; + } + + results[idx++] = ticket; + continue; + } + + if (results != null) { + completed(results); + return null; + } + } + + assert results != null; + return compositeResult(results); + } + + private int completed(Ticket[] results) { + int remaining = 1; + for (Ticket ticket : results) { + if (ticket == null) { + break; + } + int res = ticket.completed(); + if (res <= 0) { + remaining = res; + } + } + return remaining; + } + + private int failed(Throwable cause, Ticket[] results) { + int remaining = 1; + for (Ticket ticket : results) { + if (ticket == null) { + break; + } + int res = ticket.failed(cause); + if (res <= 0) { + remaining = res; + } + } + return remaining; + } + + private int dropped(Ticket[] results) { + int remaining = 1; + for (Ticket ticket : results) { + if (ticket == null) { + break; + } + int res = ticket.dropped(); + if (res <= 0) { + remaining = res; + } + } + return remaining; + } + + private int cancelled(Ticket[] results) { + int remaining = 1; + for (Ticket ticket : results) { + if (ticket == null) { + break; + } + int res = ticket.ignored(); + if (res <= 0) { + remaining = res; + } + } + return remaining; + } + + private Ticket compositeResult(final Ticket[] tickets) { + return new Ticket() { + @Override + public LimiterState state() { + // Targeting the most specific one (assuming an order of rate-limiter, customer-quota-limiter + // In the future we could make this configurable if proven useful. + return tickets[tickets.length - 1].state(); + } + + @Override + public int completed() { + return CompositeCapacityLimiter.this.completed(tickets); + } + + @Override + public int failed(final Throwable cause) { + return CompositeCapacityLimiter.this.failed(cause, tickets); + } + + @Override + public int dropped() { + return CompositeCapacityLimiter.this.dropped(tickets); + } + + @Override + public int ignored() { + return CompositeCapacityLimiter.this.cancelled(tickets); + } + }; + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java new file mode 100644 index 0000000000..548040edf4 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java @@ -0,0 +1,173 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import io.servicetalk.apple.capacity.limiter.api.FixedCapacityLimiterBuilder.StateObserver; +import io.servicetalk.context.api.ContextMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import javax.annotation.Nullable; + +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.util.concurrent.atomic.AtomicIntegerFieldUpdater.newUpdater; + +final class FixedCapacityLimiter implements CapacityLimiter { + + private static final Logger LOGGER = LoggerFactory.getLogger(FixedCapacityLimiter.class); + + private static final AtomicIntegerFieldUpdater pendingUpdater = + newUpdater(FixedCapacityLimiter.class, "pending"); + + private final int capacity; + @Nullable + private final StateObserver observer; + private final String name; + + private volatile int pending; + + FixedCapacityLimiter(final int capacity) { + this(FixedCapacityLimiter.class.getSimpleName(), capacity, null); + } + + FixedCapacityLimiter(final String name, final int capacity, @Nullable StateObserver observer) { + if (capacity <= 0) { + throw new IllegalArgumentException("capacity: " + capacity + " (expected: > 0)"); + } + this.name = name; + this.capacity = capacity; + this.observer = observer == null ? null : new CatchAllStateObserver(observer); + } + + @Override + public String name() { + return name; + } + + @Override + public Ticket tryAcquire(final Classification classification, final ContextMap meta) { + final int weight = min(max(classification.weight(), 0), 100); + final int effectiveLimit = (capacity * weight) / 100; + for (;;) { + final int currPending = pending; + if (currPending == effectiveLimit && + pendingUpdater.compareAndSet(this, currPending, currPending)) { + notifyObserver(currPending); + return null; + } else if (currPending < effectiveLimit) { + final int newPending = currPending + 1; + if (pendingUpdater.compareAndSet(this, currPending, newPending)) { + notifyObserver(newPending); + return new DefaultTicket(this, effectiveLimit - newPending); + } + } + } + } + + private void notifyObserver(final int pending) { + if (observer != null) { + observer.observe(capacity, pending); + } + } + + @Override + public String toString() { + return "FixedCapacityLimiter{" + + "capacity=" + capacity + + ", name='" + name + '\'' + + ", pending=" + pending + + '}'; + } + + private static final class DefaultTicket implements Ticket, LimiterState { + + private final FixedCapacityLimiter fixedCapacityProvider; + private final int remaining; + + DefaultTicket(final FixedCapacityLimiter fixedCapacityProvider, int remaining) { + this.fixedCapacityProvider = fixedCapacityProvider; + this.remaining = remaining; + } + + @Override + public LimiterState state() { + return this; + } + + @Override + public int remaining() { + return remaining; + } + + private int release() { + final int pending = pendingUpdater.decrementAndGet(fixedCapacityProvider); + fixedCapacityProvider.notifyObserver(pending); + return max(0, fixedCapacityProvider.capacity - pending); + } + + @Override + public int completed() { + return release(); + } + + @Override + public int dropped() { + return release(); + } + + @Override + public int failed(final Throwable __) { + return release(); + } + + @Override + public int ignored() { + return release(); + } + } + + private static final class CatchAllStateObserver implements StateObserver { + + private final StateObserver delegate; + + CatchAllStateObserver(final StateObserver delegate) { + this.delegate = delegate; + } + + @Override + public void observe(final int consumed) { + try { + delegate.observe(consumed); + } catch (Throwable t) { + LOGGER.warn("Unexpected exception from {}.observe({})", + delegate.getClass().getSimpleName(), consumed, t); + } + } + + @Override + public void observe(final int capacity, final int consumed) { + try { + delegate.observe(capacity, consumed); + } catch (Throwable t) { + LOGGER.warn("Unexpected exception from {}.observe({}, {})", + delegate.getClass().getSimpleName(), capacity, consumed, t); + } + } + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java new file mode 100644 index 0000000000..4f2d3ab1c0 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java @@ -0,0 +1,113 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import java.util.concurrent.atomic.AtomicInteger; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static java.util.Objects.requireNonNull; + +/** + * A builder for fixed capacity {@link CapacityLimiter}. + */ +public final class FixedCapacityLimiterBuilder { + + private static final AtomicInteger SEQ_GEN = new AtomicInteger(); + private int capacity; + @Nullable + private String name; + @Nullable + private StateObserver observer; + + /** + * Defines a name for this {@link CapacityLimiter}. + * @param name the name to be used when building this {@link CapacityLimiter}. + * @return {@code this}. + */ + public FixedCapacityLimiterBuilder name(final String name) { + this.name = requireNonNull(name); + return this; + } + + /** + * Defines the fixed capacity for the {@link CapacityLimiter}. + * Concurrent requests above this figure will be rejected. Requests with particular + * {@link Classification#weight() weight} will be respected and the total capacity for them will be adjusted + * accordingly. + * @param capacity The max allowed concurrent requests that this {@link CapacityLimiter} should allow. + * @return {@code this}. + */ + public FixedCapacityLimiterBuilder capacity(final int capacity) { + this.capacity = capacity; + return this; + } + + /** + * A {@link StateObserver observer} to consume the current consumed capacity. + * Useful for logging, metrics, or just debugging. + *

+ * The rate of reporting limit and consumption to the observer is based on the rate of change to + * this {@link CapacityLimiter}. + *

+ * It's expected that this {@link StateObserver} is not going to block the thread that invokes it. + * @param observer The {@link StateObserver} to inform about the current consumption of + * this {@link CapacityLimiter}. + * @return {@code this}. + */ + public FixedCapacityLimiterBuilder stateObserver(final StateObserver observer) { + this.observer = requireNonNull(observer); + return this; + } + + /** + * Build a fixed capacity {@link CapacityLimiter} according to this configuration. + * @return A new instance of {@link CapacityLimiter} according to this configuration. + */ + public CapacityLimiter build() { + return new FixedCapacityLimiter(name(), capacity, observer); + } + + @Nonnull + private String name() { + return name == null ? FixedCapacityLimiter.class.getSimpleName() + "_" + SEQ_GEN.incrementAndGet() : name; + } + + /** + * A state observer for the fixed {@link CapacityLimiter} to monitor internal limit and consumption. + */ + @FunctionalInterface + public interface StateObserver { + /** + * Callback that gives access to internal state of the {@link CapacityLimiter} with fixed capacity. + * + * @param consumed The current consumption (portion of the capacity) of the limiter. + * @deprecated Use {@link #observe(int, int)}. + */ + @Deprecated // FIXME: 0.43 - remove deprecated method or change default impl + void observe(int consumed); + + /** + * Callback that gives access to internal state of the {@link CapacityLimiter} with fixed capacity. + * + * @param capacity The max allowed concurrent requests that {@link CapacityLimiter} should allow. + * @param consumed The current consumption (portion of the capacity) of the limiter. + */ + default void observe(int capacity, int consumed) { + observe(consumed); + } + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiter.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiter.java new file mode 100644 index 0000000000..0daf9d8d6e --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiter.java @@ -0,0 +1,338 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Copyright © 2018 Netflix, Inc. and the Netflix Concurrency Limits authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterBuilder.Observer; +import io.servicetalk.context.api.ContextMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.function.BiFunction; +import java.util.function.BiPredicate; +import java.util.function.LongSupplier; + +import static java.lang.Double.isNaN; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +/** + * Gradient is a dynamic concurrency limit algorithm used for clients. + *

+ * Gradient's basic concept is that it tracks two Round Trip Time (RTT) figures, one of long period and another one + * of a shorter period. These two figures are then compared, and a gradient value is produced, representing the + * change between the two. That gradient value can in-turn be used to signify load; i.e. a positive gradient can + * mean that the RTTs are decreasing, whereas a negative value means that RTTs are increasing. + * This figure can be used to deduce a new limit (lower or higher accordingly) to follow the observed load pattern. + *

+ * The algorithm is heavily influenced by the following prior-art + *

+ */ +final class GradientCapacityLimiter implements CapacityLimiter { + + private static final Logger LOGGER = LoggerFactory.getLogger(GradientCapacityLimiter.class); + + private final Object lock = new Object(); + + private final String name; + private final int min; + private final int max; + private final float backoffRatioOnLimit; + private final float backoffRatioOnLoss; + private final long limitUpdateIntervalNs; + private final float minGradient; + private final float maxPositiveGradient; + private final Observer observer; + private final BiPredicate suspendLimitInc; + private final BiPredicate suspendLimitDec; + private final BiFunction headroom; + private final LongSupplier timeSource; + private final int initial; + private final LatencyTracker longLatency; + private final LatencyTracker shortLatency; + + private int pending; + private double limit; + private long lastSamplingNs; + GradientCapacityLimiter(final String name, final int min, final int max, final int initial, + final float backoffRatioOnLimit, final float backoffRatioOnLoss, + final LatencyTracker shortLatencyTracker, final LatencyTracker longLatencyTracker, + final Duration limitUpdateInterval, final float minGradient, + final float maxPositiveGradient, final Observer observer, + final BiPredicate suspendLimitInc, + final BiPredicate suspendLimitDec, + final BiFunction headroom, final LongSupplier timeSource) { + this.name = name; + this.min = min; + this.max = max; + this.initial = initial; + this.limit = initial; + this.backoffRatioOnLimit = backoffRatioOnLimit; + this.backoffRatioOnLoss = backoffRatioOnLoss; + this.limitUpdateIntervalNs = limitUpdateInterval.toNanos(); + this.minGradient = minGradient; + this.maxPositiveGradient = maxPositiveGradient; + this.observer = new CatchAllObserver(observer); + this.suspendLimitInc = suspendLimitInc; + this.suspendLimitDec = suspendLimitDec; + this.headroom = headroom; + this.timeSource = timeSource; + this.lastSamplingNs = timeSource.getAsLong(); + this.longLatency = longLatencyTracker; + this.shortLatency = shortLatencyTracker; + } + + @Override + public String name() { + return name; + } + + @Override + public Ticket tryAcquire(final Classification classification, final ContextMap meta) { + int newPending; + int newLimit; + + Ticket ticket = null; + synchronized (lock) { + newLimit = (int) limit; + newPending = pending; + if (pending < limit) { + newPending = ++pending; + ticket = new DefaultTicket(this, newLimit - newPending); + } + } + + observer.onStateChange(newLimit, newPending); + if (ticket != null) { + observer.onActiveRequestsIncr(); + } + return ticket; + } + + /** + * Needs to be called within a synchronized block. + */ + private int updateLimit(final long timestampNs, final double shortLatencyMillis, final double longLatencyMillis) { + if (isNaN(longLatencyMillis) || isNaN(shortLatencyMillis) || shortLatencyMillis == 0) { + return -1; + } + lastSamplingNs = timestampNs; + final double gradient = max(minGradient, min(maxPositiveGradient, longLatencyMillis / shortLatencyMillis)); + // When positive gradient, and limit already above initial, + // avoid increasing the limit when we are far from meeting it - i.e. blast radius. + final boolean isPositiveSuspended = !isNaN(gradient) && + (gradient > 1.0 && limit > initial && suspendLimitInc.test(pending, limit)); + // When negative gradient, and consumption not close to limit, + // avoid decreasing the limit. Low RPS & noisy environments (RTT deviations > 500ms) tend to bring the limit + // down to "min" even though there aren't many pending requests to justify the decision. + final boolean isNegativeSuspended = !isNaN(gradient) && gradient < 1.0 && suspendLimitDec.test(pending, limit); + if (isNaN(gradient) || isPositiveSuspended || isNegativeSuspended) { + return -1; + } + + final double headroom = gradient >= 1 ? this.headroom.apply(gradient, limit) : 0; + final double oldLimit = limit; + final int newLimit = (int) (limit = min(max, max(min, (gradient * limit) + headroom))); + observer.onLimitChange(longLatencyMillis, shortLatencyMillis, gradient, oldLimit, newLimit); + return newLimit; + } + + private int onSuccess(final long durationNs) { + final long nowNs = timeSource.getAsLong(); + final long rttMillis = NANOSECONDS.toMillis(durationNs); + int newPending; + int limit; + synchronized (lock) { + limit = (int) this.limit; + final double longLatencyMillis = longLatency.observe(nowNs, rttMillis); + final double shortLatencyMillis = shortLatency.observe(nowNs, rttMillis); + + newPending = --pending; + if ((nowNs - lastSamplingNs) >= limitUpdateIntervalNs) { + limit = updateLimit(nowNs, shortLatencyMillis, longLatencyMillis); + } + } + + if (limit > -1) { + observer.onStateChange(limit, newPending); + } + + observer.onActiveRequestsDecr(); + return limit - newPending; + } + + private int onDrop() { + int newPending; + double newLimit; + + synchronized (lock) { + newLimit = limit = max(min, limit * (limit >= max ? backoffRatioOnLimit : backoffRatioOnLoss)); + newPending = --pending; + } + + observer.onActiveRequestsDecr(); + observer.onStateChange((int) newLimit, newPending); + return (int) (newLimit - newPending); + } + + private int onIgnore() { + int newPending; + double newLimit; + + synchronized (lock) { + newLimit = limit; + newPending = --pending; + } + observer.onActiveRequestsDecr(); + observer.onStateChange((int) newLimit, newPending); + return (int) (newLimit - newPending); + } + + @Override + public String toString() { + synchronized (lock) { + return "GradientCapacityLimiter{" + + ", name='" + name + '\'' + + ", min=" + min + + ", max=" + max + + ", backoffRatioOnLimit=" + backoffRatioOnLimit + + ", backoffRatioOnLoss=" + backoffRatioOnLoss + + ", limitUpdateIntervalNs=" + limitUpdateIntervalNs + + ", minGradient=" + minGradient + + ", maxPositiveGradient=" + maxPositiveGradient + + ", initial=" + initial + + ", pending=" + pending + + ", limit=" + limit + + ", lastSamplingNs=" + lastSamplingNs + + '}'; + } + } + + private static final class DefaultTicket implements Ticket, LimiterState { + + private final long startTime; + private final GradientCapacityLimiter provider; + private final int remaining; + + DefaultTicket(final GradientCapacityLimiter provider, final int remaining) { + this.provider = provider; + this.startTime = provider.timeSource.getAsLong(); + this.remaining = remaining; + } + + @Override + public LimiterState state() { + return this; + } + + @Override + public int remaining() { + return remaining; + } + + @Override + public int completed() { + return provider.onSuccess(provider.timeSource.getAsLong() - startTime); + } + + @Override + public int dropped() { + return provider.onDrop(); + } + + @Override + public int failed(Throwable __) { + return provider.onDrop(); + } + + @Override + public int ignored() { + return provider.onIgnore(); + } + } + + private static final class CatchAllObserver implements Observer { + + private final Observer delegate; + + CatchAllObserver(Observer observer) { + this.delegate = observer; + } + + @Deprecated + @Override + public void onStateChange(final int limit, final int consumed) { + try { + delegate.onStateChange(limit, consumed); + } catch (Throwable t) { + LOGGER.warn("Unexpected exception from {}.onStateChange({}, {})", + delegate.getClass().getSimpleName(), limit, consumed, t); + } + } + + @Override + public void onActiveRequestsIncr() { + try { + delegate.onActiveRequestsIncr(); + } catch (Throwable t) { + LOGGER.warn("Unexpected exception from {}.onActiveRequestsIncr()", + delegate.getClass().getSimpleName(), t); + } + } + + @Override + public void onActiveRequestsDecr() { + try { + delegate.onActiveRequestsDecr(); + } catch (Throwable t) { + LOGGER.warn("Unexpected exception from {}.onActiveRequestsDecr()", + delegate.getClass().getSimpleName(), t); + } + } + + @Override + public void onLimitChange(final double longRtt, final double shortRtt, final double gradient, + final double oldLimit, final double newLimit) { + try { + delegate.onLimitChange(longRtt, shortRtt, gradient, oldLimit, newLimit); + } catch (Throwable t) { + LOGGER.warn("Unexpected exception from {}.onLimitChange({}, {}, {}, {}, {})", + delegate.getClass().getSimpleName(), longRtt, shortRtt, gradient, oldLimit, newLimit, t); + } + } + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterBuilder.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterBuilder.java new file mode 100644 index 0000000000..e7e46df65f --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterBuilder.java @@ -0,0 +1,386 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter.Ticket; +import io.servicetalk.context.api.ContextMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; +import java.util.function.BiPredicate; +import java.util.function.LongSupplier; +import java.util.function.Predicate; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_INITIAL_LIMIT; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_LIMIT_UPDATE_INTERVAL; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_LONG_LATENCY_TRACKER; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_MAX_GRADIENT; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_MAX_LIMIT; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_MIN_GRADIENT; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_MIN_LIMIT; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_ON_DROP; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_ON_LIMIT; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_SHORT_LATENCY_TRACKER; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_SUSPEND_DEC; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_SUSPEND_INCR; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.GREEDY_HEADROOM; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.MIN_SAMPLING_DURATION; +import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkBetweenZeroAndOne; +import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkBetweenZeroAndOneExclusive; +import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkGreaterThan; +import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkPositive; +import static java.util.Objects.requireNonNull; + +/** + * Builder for the {@link GradientCapacityLimiter} capacity limiter. + */ +public final class GradientCapacityLimiterBuilder { + + private static final Logger LOGGER = LoggerFactory.getLogger(GradientCapacityLimiter.class); + + private static final AtomicInteger SEQ_GEN = new AtomicInteger(); + + private static final Observer LOGGING_OBSERVER = new Observer() { + @Override + public void onStateChange(final int limit, final int consumed) { + LOGGER.debug("GradientCapacityLimiter: limit {} consumption {}", limit, consumed); + } + + @Override + public void onActiveRequestsDecr() { + } + + @Override + public void onActiveRequestsIncr() { + } + + @Override + public void onLimitChange(final double longRtt, final double shortRtt, final double gradient, + final double oldLimit, final double newLimit) { + LOGGER.debug("GradientCapacityLimiter: longRtt {} shortRtt {} gradient {} oldLimit {} newLimit {}", + longRtt, shortRtt, gradient, oldLimit, newLimit); + } + }; + + @Nullable + private String name; + private int initial = DEFAULT_INITIAL_LIMIT; + private int min = DEFAULT_MIN_LIMIT; + private int max = DEFAULT_MAX_LIMIT; + private float onDrop = DEFAULT_ON_DROP; + private float onLimit = DEFAULT_ON_LIMIT; + private float minGradient = DEFAULT_MIN_GRADIENT; + private float maxGradient = DEFAULT_MAX_GRADIENT; + private Observer observer = LOGGING_OBSERVER; + private Duration limitUpdateInterval = DEFAULT_LIMIT_UPDATE_INTERVAL; + private BiPredicate suspendLimitInc = DEFAULT_SUSPEND_INCR; + private BiPredicate suspendLimitDec = DEFAULT_SUSPEND_DEC; + private BiFunction headroom = GREEDY_HEADROOM; + private LongSupplier timeSource = System::nanoTime; + private LatencyTracker shortLatencyTracker = DEFAULT_SHORT_LATENCY_TRACKER; + private LatencyTracker longLatencyTracker = DEFAULT_LONG_LATENCY_TRACKER; + + GradientCapacityLimiterBuilder() { + } + + /** + * Defines a name for this {@link CapacityLimiter}. + * @param name the name to be used when building this {@link CapacityLimiter}. + * @return {@code this}. + */ + public GradientCapacityLimiterBuilder name(final String name) { + this.name = requireNonNull(name); + return this; + } + + /** + * Define {@code initial}, {@code min} and {@code max} concurrency limits for this {@link CapacityLimiter}. + * The active concurrency will fluctuate between these limits starting from the {@code min} and never + * going beyond {@code max}. + *

+ * The limit translates to a concurrency figure, e.g. how many requests can be in-flight simultaneously and + * doesn't represent a constant rate (i.e. has no notion of time). + *

+ * The lower the {@code initial} or {@code min} are, the slower the ramp up will be, and the bigger it is the + * more aggressive the client will be, keep concurrently issuing {@code min} requests to meet this limit. + *

+ * Min must always be less than max, and ideally max should be greater by 10x. + * + * @param initial The initial concurrency allowed, helps with faster start. + * @param min The minimum concurrency allowed. + * @param max The maximum concurrency allowed. + * @return {@code this}. + */ + public GradientCapacityLimiterBuilder limits(final int initial, final int min, final int max) { + checkPositive("min", min); + if (initial < min || initial > max) { + throw new IllegalArgumentException("initial: " + initial + " (expected: min <= initial <= max)"); + } + if (max <= min) { + throw new IllegalArgumentException("min: " + min + ", max: " + max + " (expected: min < max)"); + } + + this.initial = initial; + this.min = min; + this.max = max; + return this; + } + + /** + * Defines the backoff ratios when certain conditions are met. + * Ratios are used to alter the limit of the {@link CapacityLimiter} by the provided multiplier on different + * conditions as identified by their name. + * + *

+ * The formula for the backoff ratio used is: {@code NewLimit = OldLimit * BackoffRatio}, always respecting the + * {@link #min} and {@link #max} values. + * + *

+ * Both limits must be between 0 and 1 exclusively. + * + * @param onDrop The backoff ratio used to bring the limit down by that amount, when a request is dropped + * either by a server response identified as a rejection, or by a local timeout. + * @param onLimit The backoff ratio used to bring the limit down by that amount, when the maximum limit is + * reached. + * @return {@code this}. + */ + public GradientCapacityLimiterBuilder backoffRatio(final float onDrop, final float onLimit) { + checkBetweenZeroAndOneExclusive("onDrop", onDrop); + checkBetweenZeroAndOneExclusive("onLimit", onLimit); + + this.onDrop = onDrop; + this.onLimit = onLimit; + return this; + } + + /** + * A function used to positively influence the new limit when the detected gradient is positive (i.e. > 1.0). + * This can be used to grow the limit faster when the algorithm is being very conservative. Through testing + * a good headroom function can be the square root of the gradient (i.e. {@link Math#sqrt(double)}). + * The input of the function is the newly calculated gradient, and the previous limit in this order. + * + * @param headroom An extra padding for the new limit if the algorithm is being very conservative. + * @return {@code this}. + */ + public GradientCapacityLimiterBuilder headroom(final BiFunction headroom) { + requireNonNull(headroom); + this.headroom = headroom; + return this; + } + + /** + * Sets the {@link LatencyTracker} for the purposes of tracking the latency changes over a shorter period of time + * (recent) as required for the Gradient algorithm to deduce changes when compared against a longer period of + * time (past). The tracker will observe (provided) all latencies observed from all requests that go through the + * {@link CapacityLimiter}, it's up to the {@link LatencyTracker} which latencies to consider and how big or small + * the trackable window will be. + * @param shortLatencyTracker The {@link LatencyTracker} to be used to track the recent latency changes + * (short window). + * @return {@code this}. + */ + GradientCapacityLimiterBuilder shortLatencyTracker(final LatencyTracker shortLatencyTracker) { + this.shortLatencyTracker = requireNonNull(shortLatencyTracker); + return this; + } + + /** + * Sets the {@link LatencyTracker} for the purposes of tracking the latency changes over a longer period of time + * (past) as required for the Gradient algorithm to deduce changes when compared against a shorter period of + * time (recent). The tracker will observe (provided) all latencies observed from all requests that go through the + * {@link CapacityLimiter}, it's up to the {@link LatencyTracker} which latencies to consider and how big or small + * the trackable window will be. + * @param longLatencyTracker The {@link LatencyTracker} to be used to track the history of latencies + * so far (long exposed window). + * @return {@code this}. + */ + GradientCapacityLimiterBuilder longLatencyTracker(final LatencyTracker longLatencyTracker) { + this.longLatencyTracker = requireNonNull(longLatencyTracker); + return this; + } + + /** + * How often a new sampled RTT must be collected to update the concurrency limit. + * This interval is part of the normal + * {@link CapacityLimiter#tryAcquire(Classification, ContextMap)} flow, thus no external + * {@link java.util.concurrent.Executor} is used. The more traffic goes through the more accurate it will be. + * As a result this is not a hard deadline, but rather an at-least figure. + *

+ * There is a hard min duration applied of {@code 50ms} that can't be overridden. Any input less than that value + * will result in a {@link IllegalArgumentException}. That minimum interval was determined experimentally to + * avoid extreme adjustments of the limit without a cool off period. + * + * @param duration The duration between sampling an RTT value. + * @return {@code this}. + */ + public GradientCapacityLimiterBuilder limitUpdateInterval(final Duration duration) { + requireNonNull(duration); + if (duration.toMillis() < MIN_SAMPLING_DURATION.toMillis()) { + throw new IllegalArgumentException("Sampling interval " + duration + " (expected > 50ms)."); + } + this.limitUpdateInterval = duration; + return this; + } + + /** + * Defines the min gradient allowance per {@link #limitUpdateInterval(Duration) sampling-interval}. + * This helps push the limit upwards by not allowing quick drops, and it tends to maintain that higher limit. + * + * @param minGradient The minimum allowed gradient per {@link #limitUpdateInterval(Duration) sampling + * interval}. + * @return {@code this}. + */ + public GradientCapacityLimiterBuilder minGradient(final float minGradient) { + checkBetweenZeroAndOne("minGradient", minGradient); + this.minGradient = minGradient; + return this; + } + + /** + * Defines the max gradient allowance per {@link #limitUpdateInterval(Duration) sampling-interval}. + * This helps limit how fast the limit can grow, allowing the peer to adjust to the change, before the + * limit grows out of control causing start-stop reactions (saw-tooth patterns in traffic). + * + * @param maxPositiveGradient The maximum allowed gradient per {@link #limitUpdateInterval(Duration) sampling + * interval}. + * @return {@code this}. + */ + public GradientCapacityLimiterBuilder maxGradient(final float maxPositiveGradient) { + checkGreaterThan("maxGradient", maxPositiveGradient, 1.0f); + this.maxGradient = maxPositiveGradient; + return this; + } + + /** + * A {@link Observer observer} to consume the current state of this {@link CapacityLimiter} when + * state changes occur. Useful to monitor the limit through logging or metrics, or just debugging. + *

+ * It's expected that this {@link Observer} is not going to block the thread that invokes it. + * @param observer The {@link Observer} to inform about the current capacity and consumption + * of this {@link CapacityLimiter}. + * @return {@code this}. + */ + public GradientCapacityLimiterBuilder observer(final Observer observer) { + this.observer = requireNonNull(observer); + return this; + } + + /** + * A function to suspend the increase of the limit when that is not consumed + * (e.g. rate of requests isn't crossing it). + * This helps prevent the limit growing to infinite, and provides faster reaction when a reduction happens. + * Additionally, this works as a "blast-radius" concept, effectively limiting spontaneous traffic surges. + * @param suspendLimitInc The {@link BiPredicate} that should return {@code true} when the limit + * should halt increasing based on current consumption (first argument as {@link Integer}) and current limit + * (second argument as {@link Double}). Helper {@link GradientCapacityLimiterUtils#blastRadius(int) API} + * to offer blast-radius type predicates. + * @return {@code this}. + */ + public GradientCapacityLimiterBuilder suspendLimitIncrease(final BiPredicate suspendLimitInc) { + this.suspendLimitInc = requireNonNull(suspendLimitInc); + return this; + } + + /** + * A function to suspend the decrease of the limit when that is not consumed + * (e.g. rate of requests isn't crossing it). + * When the monitored RTT is considerably noisy (deviations > 500ms) the limit tends to decrease faster, + * even when there is no significant utilization of it going on. This helps prevent the decrease, + * until the provided {@link Predicate} toggles it. + * @param suspendLimitDec The {@link BiPredicate} that should return {@code true} when the limit + * should halt decreasing based on current consumption (first argument as {@link Integer}) and current limit + * (second argument as {@link Double}). Helper {@link GradientCapacityLimiterUtils#occupancyFactor(float) API} + * to offer occupancy-factor type predicates. + * @return {@code this}. + */ + public GradientCapacityLimiterBuilder suspendLimitDecrease(final BiPredicate suspendLimitDec) { + this.suspendLimitDec = requireNonNull(suspendLimitDec); + return this; + } + + // For testing only + GradientCapacityLimiterBuilder timeSource(final LongSupplier timeSource) { + this.timeSource = requireNonNull(timeSource); + return this; + } + + /** + * Builds a Gradient dynamic {@link CapacityLimiter} based on config options of this builder. + * + * @return A dynamic {@link CapacityLimiter} based on the options of {@code this} builder. + */ + public CapacityLimiter build() { + return new GradientCapacityLimiter(name(), min, max, initial, onLimit, onDrop, shortLatencyTracker, + longLatencyTracker, limitUpdateInterval, minGradient, maxGradient, observer, + suspendLimitInc, suspendLimitDec, headroom, timeSource); + } + + @Nonnull + private String name() { + return name == null ? GradientCapacityLimiter.class.getSimpleName() + "_" + SEQ_GEN.incrementAndGet() : name; + } + + /** + * A state observer for Gradient {@link CapacityLimiter} to monitor internal state changes. + */ + public interface Observer { + /** + * Callback that gives access to internal state of the Gradient {@link CapacityLimiter}. + * Useful to capture all consumption changes along with the limit in use, but can be very noisy, + * since consumption changes twice in the lifecycle of a {@link Ticket}. + *

+ * The rate of reporting to the observer is based on the rate of change to this + * {@link CapacityLimiter}. + * @param limit The current limit (dynamically computed) of the limiter. + * @param consumed The current consumption (portion of the limit) of the limiter. + * @deprecated alternative for consumed available through {@link #onActiveRequestsIncr} + * and {@link #onActiveRequestsDecr}, similarly alternative for limit changes available through + * {@link #onLimitChange(double, double, double, double, double)}. + */ + @Deprecated + void onStateChange(int limit, int consumed); + + /** + * Callback that informs when the active requests increased by 1. + */ + void onActiveRequestsIncr(); + + /** + * Callback that informs when the active requests decreased by 1. + */ + void onActiveRequestsDecr(); + + /** + * A limit observer callback, to consume the trigger state of this {@link CapacityLimiter} when a limit change + * happens. Useful to monitor the limit changes and their triggers. + *

+ * The rate of reporting to the observer is based on the rate of change to this + * {@link CapacityLimiter} and the {@link #limitUpdateInterval(Duration) sampling interval}. + * @param longRtt The exponential moving average stat of request response times. + * @param shortRtt The sampled response time that triggered the limit change. + * @param gradient The response time gradient (delta) between the long exposed stat (see. longRtt) + * and the sampled response time (see. shortRtt). + * @param oldLimit The previous limit of the limiter. + * @param newLimit The current limit of the limiter. + */ + void onLimitChange(double longRtt, double shortRtt, double gradient, double oldLimit, double newLimit); + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterProfiles.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterProfiles.java new file mode 100644 index 0000000000..8409e57c56 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterProfiles.java @@ -0,0 +1,120 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import io.servicetalk.apple.capacity.limiter.api.LatencyTracker.EMA; + +import java.time.Duration; +import java.util.function.BiFunction; +import java.util.function.BiPredicate; +import java.util.function.Consumer; + +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterUtils.blastRadius; +import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterUtils.occupancyFactor; +import static java.lang.Integer.MAX_VALUE; +import static java.time.Duration.ofMinutes; +import static java.time.Duration.ofSeconds; + +/** + * Default supported gradient profiles. + */ +public final class GradientCapacityLimiterProfiles { + + static final int DEFAULT_INITIAL_LIMIT = 100; + static final int DEFAULT_MIN_LIMIT = 1; + static final int DEFAULT_MAX_LIMIT = MAX_VALUE; + static final float DEFAULT_ON_DROP = 0.5f; + static final float DEFAULT_ON_LIMIT = 0.2f; + static final float DEFAULT_MIN_GRADIENT = 0.2f; + static final float GREEDY_MIN_GRADIENT = 0.5f; + static final float DEFAULT_MAX_GRADIENT = 1.2f; + static final float GREEDY_MAX_GRADIENT = 1.8f; + static final float EXPERIMENTAL_GREEDY_ON_LIMIT = 0.9f; + static final float EXPERIMENTAL_GREEDY_ON_DROP = 0.95f; + static final float EXPERIMENTAL_GREEDY_MIN_GRADIENT = 0.90f; + static final Duration DEFAULT_LIMIT_UPDATE_INTERVAL = ofSeconds(1); + static final BiPredicate DEFAULT_SUSPEND_INCR = blastRadius(2); + static final BiPredicate DEFAULT_SUSPEND_DEC = (__, ___) -> false; + static final BiPredicate EXPERIMENTAL_SUSPEND_DEC = occupancyFactor(.9f); + static final BiFunction DEFAULT_HEADROOM = (__, ___) -> 0.0; + static final BiFunction GREEDY_HEADROOM = (grad, limit) -> Math.sqrt(grad * limit); + static final Duration MIN_SAMPLING_DURATION = Duration.ofMillis(50); + static final LatencyTracker SHORT_LATENCY_CALMER_TRACKER = new EMA(Duration.ofMillis(500).toNanos()); + static final LatencyTracker LONG_LATENCY_CALMER_TRACKER = new EMA(ofSeconds(1).toNanos()); + static final BiFunction CALMER_RATIO = + (tracker, calmer) -> calmer < (tracker / 2) ? .90f : -1f; + static final LatencyTracker DEFAULT_SHORT_LATENCY_TRACKER = new LatencyTracker.LastSample(); + static final LatencyTracker EXPERIMENTAL_SHORT_LATENCY_TRACKER = new EMA(ofSeconds(10).toNanos(), + SHORT_LATENCY_CALMER_TRACKER, CALMER_RATIO); + static final LatencyTracker DEFAULT_LONG_LATENCY_TRACKER = new EMA(ofMinutes(10).toNanos(), + LONG_LATENCY_CALMER_TRACKER, CALMER_RATIO); + + private GradientCapacityLimiterProfiles() { + // No instances + } + + /** + * The settings applied from this profile demonstrate cautious behaviour of the {@link CapacityLimiter}, + * that tries to keep the limit lower to avoid increasing the latency of requests. + * This is a suggested setting for latency sensitive applications, but be aware that it may start throttling much + * earlier when even small gradients are noticed in the response times. + * @return Settings for the {@link GradientCapacityLimiterBuilder} for a cautious Gradient {@link CapacityLimiter}. + */ + public static Consumer preferLatency() { + return builder -> + builder.minGradient(DEFAULT_MIN_GRADIENT) + .maxGradient(DEFAULT_MAX_GRADIENT) + .shortLatencyTracker(new LatencyTracker.LastSample()) + .headroom(DEFAULT_HEADROOM); + } + + /** + * The settings applied from this profile demonstrate aggressive behaviour of the {@link CapacityLimiter}, + * that tries to push the limit higher until a significant gradient change is noticed. It will allow limit increases + * while latency is changing, favouring throughput overall, so latency sensitive application may not want to use + * this profile. + * @return Settings for the {@link GradientCapacityLimiterBuilder} for an aggressive Gradient + * {@link CapacityLimiter}. + */ + public static Consumer preferThroughput() { + return builder -> + builder.minGradient(GREEDY_MIN_GRADIENT) + .maxGradient(GREEDY_MAX_GRADIENT) + .headroom(GREEDY_HEADROOM); + } + + /** + * The settings applied from this profile demonstrate aggressive behaviour of the {@link CapacityLimiter}, + * that tries to push the limit higher until a significant gradient change is noticed. It will allow limit increases + * while latency is changing, favouring throughput overall, so latency sensitive application may not want to use + * this profile. + *

+ * Note: This experimental profile is a new configuration that we are trying to collect metrics + * on the behavior and how it compares against the exiting offer. + * + * @return Settings for the {@link GradientCapacityLimiterBuilder} for an aggressive Gradient + * {@link CapacityLimiter}. + */ + public static Consumer preferThroughputExperimental() { + return builder -> + builder.minGradient(EXPERIMENTAL_GREEDY_MIN_GRADIENT) + .maxGradient(GREEDY_MAX_GRADIENT) + .backoffRatio(EXPERIMENTAL_GREEDY_ON_DROP, EXPERIMENTAL_GREEDY_ON_LIMIT) + .shortLatencyTracker(EXPERIMENTAL_SHORT_LATENCY_TRACKER) + .suspendLimitDecrease(EXPERIMENTAL_SUSPEND_DEC) + .headroom(GREEDY_HEADROOM); + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterUtils.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterUtils.java new file mode 100644 index 0000000000..d961fc7234 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterUtils.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2022 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import java.util.function.BiPredicate; + +public final class GradientCapacityLimiterUtils { + + private GradientCapacityLimiterUtils() { + // No instances + } + + /** + * Helper function to calculate a maximum limit growth when current consumption is below the current limit. + * Limits when not disturbed can grow indefinitely (to their maximum setting), which has the negative effect of + * taking more time to react to bad situations and shrink down to a healthy figure. This "blast-radius" concept + * allows the limit to grow in a controlled way (e.g., grow 2x the current need) which still allows for traffic + * spikes to happen uninterrupted, but also faster reaction times. + * @param scale the scaling factor to use a multiplier of the current consumption. + * @return A {@link BiPredicate} where the first argument represents the current consumption of the Gradient, + * and the second argument is the current limit. + */ + public static BiPredicate blastRadius(final int scale) { + return (inFlight, limit) -> (inFlight * scale) < limit; + } + + /** + * Helper function to calculate a minimum target for limit decrease when current consumption is not reaching the + * set limit. Limits can shrink to they minimum setting, when the RTT deviation is significant, even though that + * doesn't cause buildups. + * The "occupancy-factor" concept allows the limit to stay put even in presence of negative gradients, when its not + * utilised. + * @param factor the target utilization of the limit before it should start decreasing. + * @return A {@link BiPredicate} where the first argument represents the current consumption of the Gradient, + * and the second argument is the current limit. + */ + public static BiPredicate occupancyFactor(final float factor) { + return (inFlight, limit) -> inFlight <= (limit * factor); + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/LatencyTracker.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/LatencyTracker.java new file mode 100644 index 0000000000..af7405a741 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/LatencyTracker.java @@ -0,0 +1,120 @@ +/* + * Copyright © 2022 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import java.util.function.BiFunction; +import javax.annotation.Nullable; + +import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkPositive; +import static java.lang.Math.exp; +import static java.lang.Math.log; +import static java.util.Objects.requireNonNull; + +/** + * A tracker of latency values at certain points in time. + * + * This helps observe latency behavior and different implementation can offer their own interpretation of the latency + * tracker, allowing a way to enhance the algorithm's behavior according to the observations. + * Implementations must provide thread-safety guarantees. + */ +interface LatencyTracker { + + /** + * Observe a latency figure at a certain point in time. + * + * @param timestampNs The time this latency was observed (in Nanoseconds). + * @param latencyMillis The observed latency (in Milliseconds). + * @return the result of the observations so far. + */ + double observe(long timestampNs, long latencyMillis); + + /** + * A {@link LatencyTracker} that keeps track of the latest observed latency. + */ + final class LastSample implements LatencyTracker { + @Override + public double observe(long timestampNs, final long latency) { + return latency; + } + } + + /** + * Exponential weighted moving average based on the work by Andreas Eckner. + * + * @see Eckner + * (2019) Algorithms for Unevenly Spaced Time Series: Moving Averages and Other Rolling Operators (4.2 pp. 10) + */ + final class EMA implements LatencyTracker { + + private final double tau; + @Nullable + private final LatencyTracker calmTracker; + @Nullable + private final BiFunction calmRatio; + + private double ewma; + private long lastTimeNanos; + + /** + * Constructs an Exponential Moving Average {@link LatencyTracker} with decay window of halfLifeNs. + * @param halfLifeNs The decay window of the EMA. + */ + EMA(final long halfLifeNs) { + this.tau = halfLifeNs / log(2); + this.calmTracker = null; + this.calmRatio = null; + } + + /** + * Constructs an Exponential Moving Average {@link LatencyTracker} with decay window of halfLifeNs. + * It offers additional support of shrinking the observed latency when the calmTracker observations + * are different enough to trigger the predicate. + * @param halfLifeNs The decay window of the EMA. + * @param calmTracker A {@link LatencyTracker} that is used to help shrink the observations + * of this {@link LatencyTracker} faster. The observations of this tracker and the calmTracker, + * are compared in the {@link BiFunction}. + * @param calmRatio A {@link BiFunction} that evaluates the observation of this tracker against the calmer, + * and decides on shrink ration of this observation. Any negative value, results in no calming. + */ + EMA(final long halfLifeNs, final LatencyTracker calmTracker, + final BiFunction calmRatio) { + checkPositive("halfLifeNs", halfLifeNs); + this.tau = halfLifeNs / log(2); + this.calmTracker = requireNonNull(calmTracker); + this.calmRatio = requireNonNull(calmRatio); + } + + @Override + public double observe(long timestampNs, final long latency) { + if (calmTracker != null) { + assert calmRatio != null; + final double calmedEwma = calmTracker.observe(timestampNs, latency); + final float ratio = calmRatio.apply(ewma, calmedEwma); + if (ewma > 0 && ratio > 0f) { + ewma *= ratio; + lastTimeNanos = timestampNs; + return ewma; + } + } + + final double tmp = (timestampNs - lastTimeNanos) / tau; + final double w = exp(-tmp); + ewma = ewma * w + latency * (1d - w); + lastTimeNanos = timestampNs; + return ewma; + } + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Preconditions.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Preconditions.java new file mode 100644 index 0000000000..25aabbf556 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Preconditions.java @@ -0,0 +1,77 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import java.time.Duration; + +import static java.time.Duration.ZERO; + +final class Preconditions { + + private Preconditions() { + // No instances + } + + static int checkPositive(String field, int value) { + if (value <= 0) { + throw new IllegalArgumentException(field + ": " + value + " (expected: > 0)"); + } + return value; + } + + static float checkPositive(String field, float value) { + if (value <= 0.0f) { + throw new IllegalArgumentException(field + ": " + value + " (expected: > 0.0f)"); + } + return value; + } + + static float checkGreaterThan(String field, float value, final float min) { + if (value <= min) { + throw new IllegalArgumentException(field + ": " + value + " (expected: > " + min + ")"); + } + return value; + } + + static float checkBetweenZeroAndOne(String field, float value) { + if (value < 0.0f || value > 1.0f) { + throw new IllegalArgumentException(field + ": " + value + " (expected: 0.0f <= " + field + " <= 1.0f)"); + } + return value; + } + + static int checkRange(String field, int value, int min, int max) { + if (value < min || value > max) { + throw new IllegalArgumentException(field + ": " + value + + " (expected: " + min + " <= " + field + " <= " + max + ")"); + } + return value; + } + + static float checkBetweenZeroAndOneExclusive(String field, float value) { + if (value <= 0.0f || value >= 1.0f) { + throw new IllegalArgumentException(field + ": " + value + " (expected: 0.0f < " + field + " < 1.0f)"); + } + return value; + } + + static Duration checkZeroOrPositive(String field, Duration value) { + if (ZERO.compareTo(value) > 0) { + throw new IllegalArgumentException(field + ": " + value + " (expected: >= " + ZERO + ")"); + } + return value; + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/RequestRejectedException.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/RequestRejectedException.java new file mode 100644 index 0000000000..5a97e38070 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/RequestRejectedException.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import javax.annotation.Nullable; + +/** + * An {@link Exception} to indicate that a request was rejected by a client/server due to capacity constraints. + */ +public class RequestRejectedException extends RuntimeException { + + private static final long serialVersionUID = 2152182132883133067L; + + /** + * Creates a new instance. + */ + public RequestRejectedException() { + } + + /** + * Creates a new instance. + * + * @param message the detail message. + */ + public RequestRejectedException(@Nullable final String message) { + super(message); + } + + /** + * Creates a new instance. + * + * @param message the detail message. + * @param cause of this exception. + */ + public RequestRejectedException(@Nullable final String message, @Nullable final Throwable cause) { + super(message, cause); + } + + /** + * Creates a new instance. + * + * @param cause of this exception. + */ + public RequestRejectedException(@Nullable final Throwable cause) { + super(cause); + } + + /** + * Creates a new instance. + * + * @param message the detail message. + * @param cause of this exception. + * @param enableSuppression {@code true} if suppression should be enabled. + * @param writableStackTrace {@code true} if the stack trace should be writable + */ + public RequestRejectedException(@Nullable final String message, @Nullable final Throwable cause, + final boolean enableSuppression, final boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/package-info.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/package-info.java new file mode 100644 index 0000000000..f17faccfd7 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@ElementsAreNonnullByDefault +package io.servicetalk.apple.capacity.limiter.api; + +import io.servicetalk.annotations.ElementsAreNonnullByDefault; diff --git a/servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterTest.java b/servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterTest.java new file mode 100644 index 0000000000..1083084856 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterTest.java @@ -0,0 +1,148 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +class AimdCapacityLimiterTest { + private static final Logger LOGGER = LoggerFactory.getLogger(AimdCapacityLimiterTest.class); + private static final Classification DEFAULT = () -> 0; + + @Test + void testOnLimitAdaptiveness() { + CapacityLimiter capacityLimiter = CapacityLimiters.dynamicAIMD() + .limits(2, 2, 4) + .increment(1f) + .cooldown(Duration.ZERO) + .backoffRatio(.8f, .2f) + .stateObserver((limit, consumed) -> LOGGER.debug("Limit: {} consumed: {}", limit, consumed)) + .build(); + + increaseLimitAndVerify(capacityLimiter); + + // Fail one, Pending 0 - New Limit = 4 * .2 => 0.8 => max(2, 0.8) => 2 + final CapacityLimiter.Ticket ticket = capacityLimiter.tryAcquire(DEFAULT, null); + assertThat(ticket, notNullValue()); + ticket.dropped(); + + // Verify new limit + for (int i = 0; i < 2; i++) { + capacityLimiter.tryAcquire(DEFAULT, null); + } + + // Now blocked + assertThat(capacityLimiter.tryAcquire(DEFAULT, null), nullValue()); + } + + @Test + void testOnDropAdaptiveness() { + CapacityLimiter capacityLimiter = CapacityLimiters.dynamicAIMD() + .limits(2, 2, 4) + .increment(1f) + .cooldown(Duration.ZERO) + .backoffRatio(.9f, .2f) + .build(); + + increaseLimitAndVerify(capacityLimiter); + + // Fail one, Pending 0 - New Limit = 4 * .9 => 3.6 => max(2, 3.6) => 3 + final CapacityLimiter.Ticket ticket = capacityLimiter.tryAcquire(DEFAULT, null); + assertThat(ticket, notNullValue()); + ticket.dropped(); + + // Verify new limit + for (int i = 0; i < 3; i++) { + capacityLimiter.tryAcquire(DEFAULT, null); + } + + // Now blocked + assertThat(capacityLimiter.tryAcquire(DEFAULT, null), nullValue()); + } + + @Test + void concurrentEvaluation() throws InterruptedException { + final CapacityLimiter capacityLimiter = CapacityLimiters.dynamicAIMD() + .limits(70, 70, 120) + .build(); + + final ExecutorService executor = Executors.newFixedThreadPool(5); + final AtomicLong succeeded = new AtomicLong(0); + final AtomicLong failed = new AtomicLong(0); + + for (int i = 0; i < 120; i++) { + executor.submit(() -> { + final CapacityLimiter.Ticket ticket = capacityLimiter.tryAcquire(DEFAULT, null); + if (ticket != null) { + ticket.completed(); + succeeded.incrementAndGet(); + } else { + failed.incrementAndGet(); + } + }); + } + + executor.shutdown(); + if (!executor.awaitTermination(10, TimeUnit.SECONDS)) { + throw new AssertionError("Executor didn't terminate in time."); + } + + assertThat(succeeded.get(), equalTo(120L)); + } + + private static void increaseLimitAndVerify(final CapacityLimiter capacityLimiter) { + CapacityLimiter.Ticket first = capacityLimiter.tryAcquire(DEFAULT, null); + // Pending 1 + assertThat(first, notNullValue()); + + CapacityLimiter.Ticket second = capacityLimiter.tryAcquire(DEFAULT, null); + // Pending 2 + assertThat(second, notNullValue()); + + // Pending 1 - New Limit 3 + first.completed(); + + // Pending 0 - New Limit 4 + second.completed(); + + List tickets = new ArrayList<>(4); + // Verify new limit + for (int i = 0; i < 4; i++) { + tickets.add(capacityLimiter.tryAcquire(DEFAULT, null)); + } + // Pending 4 - Limit 4 + + // Finalize pending + for (CapacityLimiter.Ticket ticket : tickets) { + ticket.completed(); + } + // Pending 0 - Limit 4 + } +} diff --git a/servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterTest.java b/servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterTest.java new file mode 100644 index 0000000000..2da011da10 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterTest.java @@ -0,0 +1,87 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import org.junit.jupiter.api.Test; + +import static io.servicetalk.concurrent.internal.DeliberateException.DELIBERATE_EXCEPTION; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +class FixedCapacityLimiterTest { + private final FixedCapacityLimiter provider = new FixedCapacityLimiter(1); + + @Test + void belowCapacityThenComplete() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + ticket.completed(); + + evaluateExpectingAllow(); + } + + @Test + void belowCapacityThenFailed() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + ticket.failed(DELIBERATE_EXCEPTION); + + evaluateExpectingAllow(); + } + + @Test + void belowCapacityThenCancelled() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + assertThat(ticket.ignored(), equalTo(1)); + + evaluateExpectingAllow(); + } + + @Test + void aboveCapacityThenPendingComplete() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + evaluateExpectingReject(); + assertThat(ticket.completed(), equalTo(1)); + evaluateExpectingAllow(); + } + + @Test + void aboveCapacityThenPendingFailed() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + evaluateExpectingReject(); + assertThat(ticket.failed(DELIBERATE_EXCEPTION), equalTo(1)); + evaluateExpectingAllow(); + } + + @Test + void aboveCapacityThenPendingCancelled() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + evaluateExpectingReject(); + assertThat(ticket.ignored(), equalTo(1)); + evaluateExpectingAllow(); + } + + private CapacityLimiter.Ticket evaluateExpectingAllow() { + final CapacityLimiter.Ticket ticket = provider.tryAcquire(() -> 100, null); + assertThat("Unexpected result, expected allow.", ticket, notNullValue()); + return ticket; + } + + private void evaluateExpectingReject() { + final CapacityLimiter.Ticket ticket = provider.tryAcquire(() -> 100, null); + assertThat("Unexpected result, expected reject.", ticket, nullValue()); + } +} diff --git a/servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/FixedRangeCapacityLimiterTest.java b/servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/FixedRangeCapacityLimiterTest.java new file mode 100644 index 0000000000..fb3acba297 --- /dev/null +++ b/servicetalk-capacity-limiter-api/src/test/java/io/servicetalk/apple/capacity/limiter/api/FixedRangeCapacityLimiterTest.java @@ -0,0 +1,103 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.limiter.api; + +import org.junit.jupiter.api.Test; + +import static io.servicetalk.concurrent.internal.DeliberateException.DELIBERATE_EXCEPTION; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +class FixedRangeCapacityLimiterTest { + private final FixedCapacityLimiter provider = new FixedCapacityLimiter(2); + + @Test + void belowCapacityThenComplete() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + ticket.completed(); + + evaluateExpectingAllow(); + } + + @Test + void belowCapacityThenFailed() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + ticket.failed(DELIBERATE_EXCEPTION); + + evaluateExpectingAllow(); + } + + @Test + void belowCapacityThenCancelled() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + ticket.ignored(); + + evaluateExpectingAllow(); + } + + @Test + void aboveCapacityThenPendingComplete() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + final CapacityLimiter.Ticket critTicket = evaluateExpectingSoftReject(); + + evaluateExpectingReject(); + ticket.completed(); + critTicket.completed(); + evaluateExpectingAllow(); + } + + @Test + void aboveCapacityThenPendingFailed() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + final CapacityLimiter.Ticket critReject = evaluateExpectingSoftReject(); + + evaluateExpectingReject(); + ticket.failed(DELIBERATE_EXCEPTION); + critReject.failed(DELIBERATE_EXCEPTION); + evaluateExpectingAllow(); + } + + @Test + void aboveCapacityThenPendingCancelled() { + final CapacityLimiter.Ticket ticket = evaluateExpectingAllow(); + final CapacityLimiter.Ticket prioTicket = evaluateExpectingSoftReject(); + + evaluateExpectingReject(); + ticket.failed(DELIBERATE_EXCEPTION); + prioTicket.failed(DELIBERATE_EXCEPTION); + evaluateExpectingAllow(); + } + + private CapacityLimiter.Ticket evaluateExpectingAllow() { + final CapacityLimiter.Ticket ticket = provider.tryAcquire(() -> 50, null); + assertThat("Unexpected result, expected ticket.", ticket, notNullValue()); + return ticket; + } + + private CapacityLimiter.Ticket evaluateExpectingSoftReject() { + final CapacityLimiter.Ticket ticket = provider.tryAcquire(() -> 50, null); + assertThat("Unexpected result, expected a ticket.", ticket, nullValue()); + final CapacityLimiter.Ticket critTicket = provider.tryAcquire(() -> 100, null); + assertThat("Unexpected result, expected a ticket.", critTicket, notNullValue()); + return critTicket; + } + + private void evaluateExpectingReject() { + final CapacityLimiter.Ticket critTicket = provider.tryAcquire(() -> 100, null); + assertThat("Unexpected result, expected null.", critTicket, nullValue()); + } +} diff --git a/servicetalk-circuit-breaker-api/build.gradle b/servicetalk-circuit-breaker-api/build.gradle new file mode 100644 index 0000000000..246cb1895b --- /dev/null +++ b/servicetalk-circuit-breaker-api/build.gradle @@ -0,0 +1,21 @@ +/* + * Copyright © 2021 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: "io.servicetalk.servicetalk-gradle-plugin-internal-library" + +dependencies { + implementation project(":servicetalk-annotations") +} diff --git a/servicetalk-circuit-breaker-api/license/LICENSE.resilience4j.txt b/servicetalk-circuit-breaker-api/license/LICENSE.resilience4j.txt new file mode 100644 index 0000000000..5c304d1a4a --- /dev/null +++ b/servicetalk-circuit-breaker-api/license/LICENSE.resilience4j.txt @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/servicetalk-circuit-breaker-api/src/main/java/io/servicetalk/apple/circuit/breaker/api/CircuitBreaker.java b/servicetalk-circuit-breaker-api/src/main/java/io/servicetalk/apple/circuit/breaker/api/CircuitBreaker.java new file mode 100644 index 0000000000..315bd10d82 --- /dev/null +++ b/servicetalk-circuit-breaker-api/src/main/java/io/servicetalk/apple/circuit/breaker/api/CircuitBreaker.java @@ -0,0 +1,94 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* + * Copyright 2017 The Resilience4j Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.circuit.breaker.api; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +/** + * ServiceTalk API for a Circuit Breaker. + */ +public interface CircuitBreaker { + + /** + * Return the name of this {@link CircuitBreaker}. + * @return the name of this {@link CircuitBreaker}. + */ + String name(); + + /** + * Attempt to acquire a permit to execute a call. + * + * @return {@code true} when a permit was successfully acquired. + */ + boolean tryAcquirePermit(); + + /** + * Releases a previously {@link #tryAcquirePermit() acquired} permit, without influencing the breaker in any + * way. In other words this permit is considered ignored. + */ + void ignorePermit(); + + /** + * Track a failed call and the reason. + * + * @param duration – The elapsed time duration of the call + * @param durationUnit – The duration unit + * @param throwable – The throwable which must be recorded + */ + void onError(long duration, TimeUnit durationUnit, Throwable throwable); + + /** + * Track a successful call. + * + * @param duration The elapsed time duration of the call + * @param durationUnit The duration unit + */ + void onSuccess(long duration, TimeUnit durationUnit); + + /** + * Forcefully open the breaker, if automatic state transition is supported it should be disabled after this call. + */ + void forceOpenState(); + + /** + * Returns the circuit breaker to its original closed state and resetting any internal state (timers, metrics). + */ + void reset(); + + /** + * If the state of the breaker is open, this retrieves the remaining duration it should stay open. + * Expectation is that after this duration, the breaker shall allow/offer more permits. + * + * @return The remaining duration that the breaker will remain open. + */ + Duration remainingDurationInOpenState(); +} diff --git a/servicetalk-circuit-breaker-api/src/main/java/io/servicetalk/apple/circuit/breaker/api/package-info.java b/servicetalk-circuit-breaker-api/src/main/java/io/servicetalk/apple/circuit/breaker/api/package-info.java new file mode 100644 index 0000000000..94d7d14eb2 --- /dev/null +++ b/servicetalk-circuit-breaker-api/src/main/java/io/servicetalk/apple/circuit/breaker/api/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2021 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@ElementsAreNonnullByDefault +package io.servicetalk.apple.circuit.breaker.api; + +import io.servicetalk.annotations.ElementsAreNonnullByDefault; diff --git a/servicetalk-circuit-breaker-resilience4j/build.gradle b/servicetalk-circuit-breaker-resilience4j/build.gradle new file mode 100644 index 0000000000..9110b07e58 --- /dev/null +++ b/servicetalk-circuit-breaker-resilience4j/build.gradle @@ -0,0 +1,30 @@ +/* + * Copyright © 2021 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: "io.servicetalk.servicetalk-gradle-plugin-internal-library" + +dependencies { + implementation platform(project(":servicetalk-dependencies")) + + api "io.github.resilience4j:resilience4j-circuitbreaker:${resilience4jVersion}" + api project(":servicetalk-circuit-breaker-api") + + implementation project(":servicetalk-annotations") + + testImplementation enforcedPlatform("org.junit:junit-bom:$junit5Version") + testImplementation "org.junit.jupiter:junit-jupiter-api" + testImplementation "org.hamcrest:hamcrest" +} diff --git a/servicetalk-circuit-breaker-resilience4j/gradle/spotbugs/test-exclusions.xml b/servicetalk-circuit-breaker-resilience4j/gradle/spotbugs/test-exclusions.xml new file mode 100644 index 0000000000..a10553e0a3 --- /dev/null +++ b/servicetalk-circuit-breaker-resilience4j/gradle/spotbugs/test-exclusions.xml @@ -0,0 +1,23 @@ + + + + + + + + + diff --git a/servicetalk-circuit-breaker-resilience4j/src/main/java/io/servicetalk/apple/capacity/management/resilience4j/Resilience4jAdapters.java b/servicetalk-circuit-breaker-resilience4j/src/main/java/io/servicetalk/apple/capacity/management/resilience4j/Resilience4jAdapters.java new file mode 100644 index 0000000000..d7e2a42ed8 --- /dev/null +++ b/servicetalk-circuit-breaker-resilience4j/src/main/java/io/servicetalk/apple/capacity/management/resilience4j/Resilience4jAdapters.java @@ -0,0 +1,118 @@ +/* + * Copyright © 2021 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.management.resilience4j; + +import io.servicetalk.apple.circuit.breaker.api.CircuitBreaker; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +import static io.github.resilience4j.circuitbreaker.CircuitBreaker.State.OPEN; +import static io.github.resilience4j.circuitbreaker.event.CircuitBreakerEvent.Type.STATE_TRANSITION; +import static java.lang.Math.max; +import static java.lang.System.nanoTime; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +/** + * Set of adapters converting from Resilience4j APIs to ServiceTalk APIs. + */ +public final class Resilience4jAdapters { + + private static final int IGNORE_RETRIES_ARG = 1; + + private Resilience4jAdapters() { + // No instances. + } + + /** + * ServiceTalk Circuit Breaker adapter for Resilience4j's + * {@link io.github.resilience4j.circuitbreaker.CircuitBreaker}. + * The {@code breaker} can be mutated outside the boundaries of the + * {@link io.servicetalk.apple.circuit.breaker.api.CircuitBreaker} API, allowing + * users to manually reset it or even transition states if needed. + * + * @param breaker The {@link io.github.resilience4j.circuitbreaker.CircuitBreaker} that will be adapted as a + * {@link io.servicetalk.apple.circuit.breaker.api.CircuitBreaker}. + * @return A {@link io.servicetalk.apple.circuit.breaker.api.CircuitBreaker} adapter of the provided + * {@code breaker}. + * @see Resilience4j CircuitBreaker + */ + public static io.servicetalk.apple.circuit.breaker.api.CircuitBreaker fromCircuitBreaker( + final io.github.resilience4j.circuitbreaker.CircuitBreaker breaker) { + return new R4jCircuitBreaker(requireNonNull(breaker)); + } + + private static final class R4jCircuitBreaker implements CircuitBreaker { + + private final io.github.resilience4j.circuitbreaker.CircuitBreaker breaker; + + private long breakerOpenedAtMillis; + + private R4jCircuitBreaker(final io.github.resilience4j.circuitbreaker.CircuitBreaker breaker) { + this.breaker = breaker; + this.breaker.getEventPublisher().onStateTransition(event -> { + if (STATE_TRANSITION.equals(event.getEventType()) && + OPEN.equals(event.getStateTransition().getToState())) { + breakerOpenedAtMillis = NANOSECONDS.toMillis(nanoTime()); + } + }); + } + + @Override + public String name() { + return breaker.getName(); + } + + @Override + public boolean tryAcquirePermit() { + return breaker.tryAcquirePermission(); + } + + @Override + public void ignorePermit() { + breaker.releasePermission(); + } + + @Override + public void onError(final long duration, final TimeUnit durationUnit, final Throwable throwable) { + breaker.onError(duration, durationUnit, throwable); + } + + @Override + public void onSuccess(final long duration, final TimeUnit durationUnit) { + breaker.onSuccess(duration, durationUnit); + } + + @Override + public void forceOpenState() { + breaker.transitionToForcedOpenState(); + } + + @Override + public void reset() { + breaker.reset(); + } + + @Override + public Duration remainingDurationInOpenState() { + final long openDurationMillis = breaker.getCircuitBreakerConfig() + .getWaitIntervalFunctionInOpenState().apply(IGNORE_RETRIES_ARG); + final long openDeadline = breakerOpenedAtMillis + openDurationMillis; + return Duration.ofMillis(max(0, openDeadline - NANOSECONDS.toMillis(nanoTime()))); + } + } +} diff --git a/servicetalk-circuit-breaker-resilience4j/src/main/java/io/servicetalk/apple/capacity/management/resilience4j/package-info.java b/servicetalk-circuit-breaker-resilience4j/src/main/java/io/servicetalk/apple/capacity/management/resilience4j/package-info.java new file mode 100644 index 0000000000..8625a9fda7 --- /dev/null +++ b/servicetalk-circuit-breaker-resilience4j/src/main/java/io/servicetalk/apple/capacity/management/resilience4j/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2021 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@ElementsAreNonnullByDefault +package io.servicetalk.apple.capacity.management.resilience4j; + +import io.servicetalk.annotations.ElementsAreNonnullByDefault; diff --git a/servicetalk-circuit-breaker-resilience4j/src/test/java/io/servicetalk/apple/capacity/management/resilience4j/Resilience4jAdaptersTest.java b/servicetalk-circuit-breaker-resilience4j/src/test/java/io/servicetalk/apple/capacity/management/resilience4j/Resilience4jAdaptersTest.java new file mode 100644 index 0000000000..2d1593d318 --- /dev/null +++ b/servicetalk-circuit-breaker-resilience4j/src/test/java/io/servicetalk/apple/capacity/management/resilience4j/Resilience4jAdaptersTest.java @@ -0,0 +1,81 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.capacity.management.resilience4j; + +import io.servicetalk.apple.circuit.breaker.api.CircuitBreaker; + +import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static java.lang.System.nanoTime; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class Resilience4jAdaptersTest { + + io.github.resilience4j.circuitbreaker.CircuitBreaker r4jBreaker; + CircuitBreaker breaker; + + @BeforeEach + void setup() { + CircuitBreakerConfig config = CircuitBreakerConfig + .custom() + .minimumNumberOfCalls(2) + .failureRateThreshold(50) + .currentTimestampFunction(clock -> nanoTime(), NANOSECONDS) + .build(); + + r4jBreaker = io.github.resilience4j.circuitbreaker.CircuitBreaker.of("foobar", config); + breaker = Resilience4jAdapters.fromCircuitBreaker(r4jBreaker); + } + + @Test + void verifyOpenStateDueToFailureRate() { + // 1st request allowed + assertMakeRequestAllowed(); + // Failures increased by 1 + breaker.onError(10, MILLISECONDS, new Exception()); + + // 2nd request allowed + assertMakeRequestAllowed(); + // Failures increased to 2. Min number of calls reached, and we have 100% failure rate + breaker.onError(10, MILLISECONDS, new Exception()); + + // All calls should now be rejected + assertMakeRequestRejected(); + } + + @Test + void verifyOpenStateDueToManualOpen() { + r4jBreaker.transitionToForcedOpenState(); + // All calls should now be rejected + assertMakeRequestRejected(); + + r4jBreaker.transitionToClosedState(); + assertMakeRequestAllowed(); + } + + private void assertMakeRequestAllowed() { + assertTrue(breaker.tryAcquirePermit()); + } + + private void assertMakeRequestRejected() { + assertFalse(breaker.tryAcquirePermit()); + } +} diff --git a/servicetalk-traffic-resilience-http/build.gradle b/servicetalk-traffic-resilience-http/build.gradle new file mode 100644 index 0000000000..b62b6824a3 --- /dev/null +++ b/servicetalk-traffic-resilience-http/build.gradle @@ -0,0 +1,46 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +apply plugin: "io.servicetalk.servicetalk-gradle-plugin-internal-library" + +dependencies { + implementation platform(project(":servicetalk-dependencies")) + + api project(":servicetalk-capacity-limiter-api") + api project(":servicetalk-circuit-breaker-api") + api project(":servicetalk-http-api") + api project(":servicetalk-http-utils") + api project(":servicetalk-http-netty") + + implementation project(":servicetalk-annotations") + implementation project(":servicetalk-concurrent-internal") + implementation project(":servicetalk-utils-internal") + implementation "com.google.code.findbugs:jsr305" + implementation "org.slf4j:slf4j-api" + + testImplementation enforcedPlatform("org.junit:junit-bom:$junit5Version") + testImplementation testFixtures(project(":servicetalk-concurrent-api")) + testImplementation testFixtures(project(":servicetalk-concurrent-internal")) + testImplementation testFixtures(project(":servicetalk-http-netty")) + testImplementation testFixtures(project(":servicetalk-transport-netty-internal")) + testImplementation project(":servicetalk-concurrent-api-test") + testImplementation project(":servicetalk-concurrent-test-internal") + testImplementation project(":servicetalk-http-netty") + testImplementation project(":servicetalk-test-resources") + testImplementation "org.junit.jupiter:junit-jupiter-api" + testImplementation "org.junit.jupiter:junit-jupiter-params" + testImplementation "org.hamcrest:hamcrest" + testImplementation "org.mockito:mockito-core:$mockitoCoreVersion" +} diff --git a/servicetalk-traffic-resilience-http/gradle/spotbugs/test-exclusions.xml b/servicetalk-traffic-resilience-http/gradle/spotbugs/test-exclusions.xml new file mode 100644 index 0000000000..a10553e0a3 --- /dev/null +++ b/servicetalk-traffic-resilience-http/gradle/spotbugs/test-exclusions.xml @@ -0,0 +1,23 @@ + + + + + + + + + diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/AbstractTrafficManagementHttpFilter.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/AbstractTrafficManagementHttpFilter.java new file mode 100644 index 0000000000..4ac849ec29 --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/AbstractTrafficManagementHttpFilter.java @@ -0,0 +1,373 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter; +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter.Ticket; +import io.servicetalk.apple.capacity.limiter.api.Classification; +import io.servicetalk.apple.capacity.limiter.api.RequestRejectedException; +import io.servicetalk.apple.circuit.breaker.api.CircuitBreaker; +import io.servicetalk.apple.traffic.resilience.http.PeerCapacityRejectionPolicy.PassthroughRequestRejectedException; +import io.servicetalk.apple.traffic.resilience.http.TrafficResiliencyObserver.TicketObserver; +import io.servicetalk.concurrent.api.Single; +import io.servicetalk.concurrent.api.TerminalSignalConsumer; +import io.servicetalk.context.api.ContextMap; +import io.servicetalk.http.api.HttpExecutionStrategy; +import io.servicetalk.http.api.HttpExecutionStrategyInfluencer; +import io.servicetalk.http.api.HttpRequestMetaData; +import io.servicetalk.http.api.HttpResponseMetaData; +import io.servicetalk.http.api.StreamingHttpRequest; +import io.servicetalk.http.api.StreamingHttpResponse; +import io.servicetalk.http.api.StreamingHttpResponseFactory; +import io.servicetalk.http.utils.BeforeFinallyHttpOperator; +import io.servicetalk.transport.api.ServerListenContext; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; +import javax.annotation.Nullable; + +import static io.servicetalk.concurrent.api.Single.defer; +import static io.servicetalk.concurrent.internal.ThrowableUtils.unknownStackTrace; +import static io.servicetalk.http.api.HttpExecutionStrategies.offloadNone; +import static java.lang.System.nanoTime; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.atomic.AtomicIntegerFieldUpdater.newUpdater; + +abstract class AbstractTrafficManagementHttpFilter implements HttpExecutionStrategyInfluencer { + private static final RequestRejectedException CAPACITY_REJECTION = unknownStackTrace( + new RequestRejectedException("Service under heavy load", null, false, true), + AbstractTrafficManagementHttpFilter.class, "remoteRejection"); + private static final RequestRejectedException BREAKER_REJECTION = unknownStackTrace( + new RequestRejectedException("Service Unavailable", null, false, true), + AbstractTrafficManagementHttpFilter.class, "breakerRejection"); + + protected static final Single DEFAULT_CAPACITY_REJECTION = + Single.failed(CAPACITY_REJECTION); + + protected static final Single DEFAULT_BREAKER_REJECTION = + Single.failed(BREAKER_REJECTION); + + private final Supplier> capacityPartitionsSupplier; + + private final Consumer onSuccessTicketTerminal; + + private final Consumer onCancellationTicketTerminal; + + private final BiConsumer onErrorTicketTerminal; + + private final boolean rejectWhenNotMatchedCapacityPartition; + + private final Function classifier; + + private final Predicate capacityRejectionPredicate; + + private final Predicate breakerRejectionPredicate; + + private final Supplier> circuitBreakerPartitionsSupplier; + + private final TrafficResiliencyObserver observer; + + AbstractTrafficManagementHttpFilter( + final Supplier> capacityPartitionsSupplier, + final boolean rejectWhenNotMatchedCapacityPartition, + final Function classifier, + final Predicate capacityRejectionPredicate, + final Predicate breakerRejectionPredicate, + final Consumer onSuccessTicketTerminal, + final Consumer onCancellationTicketTerminal, + final BiConsumer onErrorTicketTerminal, + final Supplier> circuitBreakerPartitionsSupplier, + final TrafficResiliencyObserver observer) { + this.capacityPartitionsSupplier = requireNonNull(capacityPartitionsSupplier, "capacityPartitionsSupplier"); + this.rejectWhenNotMatchedCapacityPartition = rejectWhenNotMatchedCapacityPartition; + this.capacityRejectionPredicate = requireNonNull(capacityRejectionPredicate, "capacityRejectionPredicate"); + this.breakerRejectionPredicate = requireNonNull(breakerRejectionPredicate, "breakerRejectionPredicate"); + this.classifier = requireNonNull(classifier, "classifier"); + this.onSuccessTicketTerminal = requireNonNull(onSuccessTicketTerminal, "onSuccessTicketTerminal"); + this.onCancellationTicketTerminal = requireNonNull(onCancellationTicketTerminal, + "onCancellationTicketTerminal"); + this.onErrorTicketTerminal = requireNonNull(onErrorTicketTerminal, "onErrorTicketTerminal"); + this.circuitBreakerPartitionsSupplier = requireNonNull(circuitBreakerPartitionsSupplier, + "circuitBreakerPartitionsSupplier"); + this.observer = requireNonNull(observer, "observer"); + } + + @Override + public HttpExecutionStrategy requiredOffloads() { + return offloadNone(); + } + + // Each filter needs a new capacity & breaker partitions functions. + final Function newCapacityPartitions() { + return capacityPartitionsSupplier.get(); + } + + final Function newCircuitBreakerPartitions() { + return circuitBreakerPartitionsSupplier.get(); + } + + Single applyCapacityControl( + final Function capacityPartitions, + final Function circuitBreakerPartitions, + @Nullable final ServerListenContext serverListenContext, + final StreamingHttpRequest request, + @Nullable final StreamingHttpResponseFactory responseFactory, + final Function> delegate) { + return defer(() -> { + final long startTime = nanoTime(); + final CapacityLimiter partition = capacityPartitions.apply(request); + if (partition == null) { + observer.onRejectedUnmatchedPartition(request); + return rejectWhenNotMatchedCapacityPartition ? + handleLocalCapacityRejection(null, request, responseFactory) + .shareContextOnSubscribe() : + handlePassthrough(delegate, request) + .shareContextOnSubscribe(); + } + + final CircuitBreaker breaker = circuitBreakerPartitions.apply(request); + final ContextMap meta = request.context(); + final Classification classification = classifier.apply(request); + Ticket ticket = partition.tryAcquire(classification, meta); + + if (ticket != null) { + ticket = new TrackingDelegatingTicket(ticket, request.hashCode()); + } + + if (ticket == null) { + observer.onRejectedLimit(request, partition.name(), meta, classification); + return handleLocalCapacityRejection(serverListenContext, request, responseFactory) + .shareContextOnSubscribe(); + } else if (breaker != null && !breaker.tryAcquirePermit()) { + observer.onRejectedOpenCircuit(request, breaker.name(), meta, classification); + // Ignore the acquired ticket if breaker was open. + ticket.ignored(); + return handleLocalBreakerRejection(request, responseFactory, breaker).shareContextOnSubscribe(); + } + + // Ticket lifetime must be completed at all points now, try/catch to ensure if anything throws (e.g. + // reactive flow isn't followed) we still complete ticket lifetime. + try { + final TicketObserver ticketObserver = observer.onAllowedThrough(request, ticket.state()); + assert ticketObserver != null; + return handleAllow(delegate, request, wrapTicket(serverListenContext, ticket), ticketObserver, + breaker, startTime).shareContextOnSubscribe(); + } catch (Throwable cause) { + onError(cause, breaker, startTime, ticket); + throw cause; + } + }); + } + + protected Ticket wrapTicket(@Nullable final ServerListenContext serverListenContext, final Ticket ticket) { + return ticket; + } + + protected abstract Single handleLocalCapacityRejection( + @Nullable ServerListenContext serverListenContext, + StreamingHttpRequest request, + @Nullable StreamingHttpResponseFactory responseFactory); + + protected abstract Single handleLocalBreakerRejection( + StreamingHttpRequest request, + @Nullable StreamingHttpResponseFactory responseFactory, + @Nullable CircuitBreaker breaker); + + RuntimeException peerCapacityRejection(final StreamingHttpResponse resp) { + return CAPACITY_REJECTION; + } + + RuntimeException peerBreakerRejection(final HttpResponseMetaData resp, final CircuitBreaker breaker) { + return BREAKER_REJECTION; + } + + private static Single handlePassthrough( + final Function> delegate, + final StreamingHttpRequest request) { + return delegate.apply(request); + } + + private Single handleAllow( + final Function> delegate, + final StreamingHttpRequest request, final Ticket ticket, final TicketObserver ticketObserver, + @Nullable final CircuitBreaker breaker, final long startTimeNs) { + return delegate.apply(request) + // The map is issuing an exception that will be propagated to the downstream BeforeFinallyHttpOperator + // in order to invoke the appropriate callbacks to release resources. + // If the BeforeFinallyHttpOperator comes earlier, the Single will succeed and only downstream will + // see the exception, preventing callbacks to release permits; which results in the limiter eventually + // throttling ALL future requests. + // Before returning an error, we have to drain the response payload body to properly release resources + // and avoid leaking a connection, except for the PassthroughRequestRejectedException case. + .flatMap(resp -> { + if (breaker != null && breakerRejectionPredicate.test(resp)) { + return resp.payloadBody().ignoreElements() + .concat(Single.failed(peerBreakerRejection(resp, breaker))) + .shareContextOnSubscribe(); + } else if (capacityRejectionPredicate.test(resp)) { + final RuntimeException rejection = peerCapacityRejection(resp); + if (PassthroughRequestRejectedException.class.equals(rejection.getClass())) { + return Single.failed(rejection).shareContextOnSubscribe(); + } + return resp.payloadBody().ignoreElements() + .concat(Single.failed(rejection)) + .shareContextOnSubscribe(); + } + return Single.succeeded(resp).shareContextOnSubscribe(); + }) + .liftSync(new BeforeFinallyHttpOperator(new TerminalSignalConsumer() { + @Override + public void onComplete() { + try { + if (breaker != null) { + breaker.onSuccess(nanoTime() - startTimeNs, NANOSECONDS); + } + } finally { + onSuccessTicketTerminal.accept(ticket); + if (ticketObserver != null) { + ticketObserver.onComplete(); + } + } + } + + @Override + public void onError(final Throwable throwable) { + AbstractTrafficManagementHttpFilter.this.onError(throwable, breaker, startTimeNs, ticket); + if (ticketObserver != null) { + ticketObserver.onError(throwable); + } + } + + @Override + public void cancel() { + try { + if (breaker != null) { + breaker.ignorePermit(); + } + } finally { + onCancellationTicketTerminal.accept(ticket); + if (ticketObserver != null) { + ticketObserver.onCancel(); + } + } + } + }, true)); + } + + private void onError(final Throwable throwable, @Nullable final CircuitBreaker breaker, + final long startTimeNs, final Ticket ticket) { + try { + if (breaker != null && !CAPACITY_REJECTION.equals(throwable)) { + // Capacity rejections should not count towards circuit-breaker stats + breaker.onError(nanoTime() - startTimeNs, NANOSECONDS, throwable); + } + } finally { + onErrorTicketTerminal.accept(ticket, throwable); + } + } + + /** + * A ticket which delegates the actual calls to the delegate, but also tracks if the actual terminal signals + * are called. + */ + static final class TrackingDelegatingTicket implements Ticket { + + private static final Logger LOGGER = LoggerFactory.getLogger(TrackingDelegatingTicket.class); + + private static final int NOT_SIGNALED = 0; + private static final int SIGNAL_COMPLETED = 1; + private static final int SIGNAL_DROPPED = 2; + private static final int SIGNAL_FAILED = 4; + private static final int SIGNAL_IGNORED = 8; + + private static final AtomicIntegerFieldUpdater signaledUpdater = + newUpdater(TrackingDelegatingTicket.class, "signaled"); + + private final Ticket delegate; + + /** + * Contains the hash code of the request which acquired this ticket. + */ + private final int requestHashCode; + + private volatile int signaled; + + TrackingDelegatingTicket(final Ticket delegate, final int requestHashCode) { + this.delegate = delegate; + this.requestHashCode = requestHashCode; + } + + @Override + public CapacityLimiter.LimiterState state() { + return delegate.state(); + } + + @Override + public int completed() { + signal(SIGNAL_COMPLETED); + return delegate.completed(); + } + + @Override + public int dropped() { + signal(SIGNAL_DROPPED); + return delegate.dropped(); + } + + @Override + public int failed(final Throwable error) { + signal(SIGNAL_FAILED); + return delegate.failed(error); + } + + @Override + public int ignored() { + signal(SIGNAL_IGNORED); + return delegate.ignored(); + } + + private void signal(final int newSignal) { + for (;;) { + final int oldValue = signaled; + if (signaledUpdater.compareAndSet(this, oldValue, oldValue | newSignal)) { + if (oldValue > NOT_SIGNALED) { + // We have a double signal, log this event since it is not expected. + LOGGER.warn("{} signaled completion more than once. Already signaled with {}, new signal {}.", + getClass().getSimpleName(), oldValue, newSignal); + } + return; + } + } + } + + @Override + public String toString() { + return "TrackingDelegatingTicket{" + + "delegate=" + delegate + + ", requestHashCode=" + requestHashCode + + ", signaled=" + signaled + + '}'; + } + } +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/DelayedRetryRequestRejectedException.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/DelayedRetryRequestRejectedException.java new file mode 100644 index 0000000000..ac0df2d2d6 --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/DelayedRetryRequestRejectedException.java @@ -0,0 +1,103 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.RequestRejectedException; +import io.servicetalk.http.api.HttpResponseStatus; +import io.servicetalk.http.netty.RetryingHttpRequesterFilter; + +import java.time.Duration; +import javax.annotation.Nullable; + +import static java.util.Objects.requireNonNull; + +/** + * A {@link RuntimeException} to indicate that a request was rejected by a server due to capacity constraints. + * This error reflects the client side application logic and its interpretation of a service response; meaning that + * its up to the application to declare whether a {@link HttpResponseStatus#TOO_MANY_REQUESTS} is a safe-to-retry + * response, and if so after how much {@link #delay()}. + */ +public final class DelayedRetryRequestRejectedException extends RequestRejectedException + implements RetryingHttpRequesterFilter.DelayedRetry { + + private static final long serialVersionUID = -7933994513110803151L; + private final Duration delay; + + /** + * Creates a new instance. + * + * @param delay The delay to be provided as input to a retry mechanism. + */ + public DelayedRetryRequestRejectedException(final Duration delay) { + this.delay = requireNonNull(delay); + } + + /** + * Creates a new instance. + * + * @param delay The delay to be provided as input to a retry mechanism. + * @param message the detail message. + */ + public DelayedRetryRequestRejectedException(final Duration delay, @Nullable final String message) { + super(message); + this.delay = requireNonNull(delay); + } + + /** + * Creates a new instance. + * + * @param delay The delay to be provided as input to a retry mechanism. + * @param message the detail message. + * @param cause of this exception. + */ + public DelayedRetryRequestRejectedException(final Duration delay, + @Nullable final String message, @Nullable final Throwable cause) { + super(message, cause); + this.delay = requireNonNull(delay); + } + + /** + * Creates a new instance. + * + * @param delay The delay to be provided as input to a retry mechanism. + * @param cause of this exception. + */ + public DelayedRetryRequestRejectedException(final Duration delay, @Nullable final Throwable cause) { + super(cause); + this.delay = requireNonNull(delay); + } + + /** + * Creates a new instance. + * + * @param delay The delay to be provided as input to a retry mechanism. + * @param message the detail message. + * @param cause of this exception. + * @param enableSuppression {@code true} if suppression should be enabled. + * @param writableStackTrace {@code true} if the stack trace should be writable + */ + public DelayedRetryRequestRejectedException(final Duration delay, + @Nullable final String message, @Nullable final Throwable cause, + final boolean enableSuppression, final boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + this.delay = requireNonNull(delay); + } + + @Override + public Duration delay() { + return delay; + } +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/NoOpTrafficResiliencyObserver.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/NoOpTrafficResiliencyObserver.java new file mode 100644 index 0000000000..ba8d160204 --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/NoOpTrafficResiliencyObserver.java @@ -0,0 +1,72 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter; +import io.servicetalk.apple.capacity.limiter.api.Classification; +import io.servicetalk.context.api.ContextMap; +import io.servicetalk.http.api.StreamingHttpRequest; + +import javax.annotation.Nullable; + +final class NoOpTrafficResiliencyObserver implements TrafficResiliencyObserver { + + static final NoOpTrafficResiliencyObserver INSTANCE = new NoOpTrafficResiliencyObserver(); + static final TicketObserver NO_OP_TICKET_OBSERVER = new NoOpTicketObserver(); + + private NoOpTrafficResiliencyObserver() { + // single instance + } + + @Override + public void onRejectedUnmatchedPartition(final StreamingHttpRequest request) { + } + + @Override + public void onRejectedLimit(final StreamingHttpRequest request, final String limiterName, + final ContextMap meta, final Classification classification) { + } + + @Override + public void onRejectedOpenCircuit(final StreamingHttpRequest request, final String breakerName, + final ContextMap meta, final Classification classification) { + } + + @Nullable + @Override + public TicketObserver onAllowedThrough(final StreamingHttpRequest request, + @Nullable final CapacityLimiter.LimiterState state) { + return NO_OP_TICKET_OBSERVER; + } + + static final class NoOpTicketObserver implements TicketObserver { + + private NoOpTicketObserver() { + } + + @Override + public void onComplete() { + } + + @Override + public void onCancel() { + } + + @Override + public void onError(final Throwable throwable) { + } + } +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/PeerCapacityRejectionPolicy.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/PeerCapacityRejectionPolicy.java new file mode 100644 index 0000000000..5fcf7cb1a0 --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/PeerCapacityRejectionPolicy.java @@ -0,0 +1,128 @@ +/* + * Copyright © 2022 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter; +import io.servicetalk.apple.capacity.limiter.api.RequestRejectedException; +import io.servicetalk.http.api.HttpResponseMetaData; +import io.servicetalk.http.api.StreamingHttpResponse; + +import java.time.Duration; +import java.util.function.Function; +import java.util.function.Predicate; + +import static io.servicetalk.apple.traffic.resilience.http.PeerCapacityRejectionPolicy.Type.REJECT; +import static io.servicetalk.apple.traffic.resilience.http.PeerCapacityRejectionPolicy.Type.REJECT_PASSTHROUGH; +import static io.servicetalk.apple.traffic.resilience.http.PeerCapacityRejectionPolicy.Type.REJECT_RETRY; +import static java.time.Duration.ZERO; +import static java.util.Objects.requireNonNull; + +/** + * Policy for peer capacity rejections that allows customization of behavior (retries or pass-through). + */ +public final class PeerCapacityRejectionPolicy { + + enum Type { + REJECT, + REJECT_PASSTHROUGH, + REJECT_RETRY, + } + + private final Predicate predicate; + private final Type type; + private final Function delayProvider; + + private PeerCapacityRejectionPolicy(final Predicate predicate, + final Type type) { + this.predicate = predicate; + this.type = type; + this.delayProvider = __ -> ZERO; + } + + PeerCapacityRejectionPolicy(final Predicate predicate, + final Type type, + final Function delayProvider) { + this.predicate = predicate; + this.type = type; + this.delayProvider = delayProvider; + } + + /** + * Evaluate responses with the given {@link Predicate} as capacity related rejections, that will affect the + * {@link CapacityLimiter} in use, but allow the original response from the upstream to pass-through this filter. + * @param predicate The {@link Predicate} to evaluate responses. + * Returning true from this {@link Predicate} signifies that the response was capacity + * related rejection from the peer. + * @return A {@link PeerCapacityRejectionPolicy}. + */ + public static PeerCapacityRejectionPolicy ofPassthrough(final Predicate predicate) { + return new PeerCapacityRejectionPolicy(predicate, REJECT_PASSTHROUGH); + } + + /** + * Evaluate responses with the given {@link Predicate} as capacity related rejections, that will affect the + * {@link CapacityLimiter} in use, and translate that to en exception. + * @param rejectionPredicate The {@link Predicate} to evaluate responses. + * Returning true from this {@link Predicate} signifies that the response was capacity + * related rejection from the peer. + * @return A {@link PeerCapacityRejectionPolicy}. + */ + public static PeerCapacityRejectionPolicy ofRejection( + final Predicate rejectionPredicate) { + return new PeerCapacityRejectionPolicy(rejectionPredicate, REJECT); + } + + /** + * Evaluate responses with the given {@link Predicate} as capacity related rejections, that will affect the + * {@link CapacityLimiter} in use, and translate that to an exception that contains "delay" information useful when + * retrying it through a retrying filter. + * @param rejectionPredicate The {@link Predicate} to evaluate responses. + * Returning true from this {@link Predicate} signifies that the response was capacity + * related rejection from the peer. + * @param delayProvider A {@link Duration} provider for delay purposes when retrying. + * @return A {@link PeerCapacityRejectionPolicy}. + */ + public static PeerCapacityRejectionPolicy ofRejectionWithRetries( + final Predicate rejectionPredicate, + final Function delayProvider) { + return new PeerCapacityRejectionPolicy(rejectionPredicate, REJECT_RETRY, delayProvider); + } + + Predicate predicate() { + return predicate; + } + + Type type() { + return type; + } + + Function delayProvider() { + return delayProvider; + } + + static final class PassthroughRequestRejectedException extends RequestRejectedException { + private static final long serialVersionUID = 5494523265208777384L; + private final StreamingHttpResponse response; + PassthroughRequestRejectedException(final String msg, final StreamingHttpResponse response) { + super(msg); + this.response = requireNonNull(response); + } + + StreamingHttpResponse response() { + return response; + } + } +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/RetryableRequestRejectedException.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/RetryableRequestRejectedException.java new file mode 100644 index 0000000000..046f1f21ed --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/RetryableRequestRejectedException.java @@ -0,0 +1,80 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.RequestRejectedException; +import io.servicetalk.transport.api.RetryableException; + +import javax.annotation.Nullable; + +/** + * A {@link RetryableException} to indicate that a request was rejected by a client/server due to capacity constraints. + * Instances of this exception are expected to be thrown when a client side capacity is reached, thus the exception did + * not touch the "wire" (network) yet, meaning that its safe to be retried. Retries are useful in the context of + * capacity, to maximize chances for a request to succeed. + */ +public final class RetryableRequestRejectedException extends RequestRejectedException + implements RetryableException { + + private static final long serialVersionUID = -1968209429496611665L; + + /** + * Creates a new instance. + */ + public RetryableRequestRejectedException() { + } + + /** + * Creates a new instance. + * + * @param message the detail message. + */ + public RetryableRequestRejectedException(@Nullable final String message) { + super(message); + } + + /** + * Creates a new instance. + * + * @param message the detail message. + * @param cause of this exception. + */ + public RetryableRequestRejectedException(@Nullable final String message, @Nullable final Throwable cause) { + super(message, cause); + } + + /** + * Creates a new instance. + * + * @param cause of this exception. + */ + public RetryableRequestRejectedException(@Nullable final Throwable cause) { + super(cause); + } + + /** + * Creates a new instance. + * + * @param message the detail message. + * @param cause of this exception. + * @param enableSuppression {@code true} if suppression should be enabled. + * @param writableStackTrace {@code true} if the stack trace should be writable + */ + public RetryableRequestRejectedException(@Nullable final String message, @Nullable final Throwable cause, + final boolean enableSuppression, final boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/SafeTrafficResiliencyObserver.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/SafeTrafficResiliencyObserver.java new file mode 100644 index 0000000000..3e631adaa0 --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/SafeTrafficResiliencyObserver.java @@ -0,0 +1,79 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter; +import io.servicetalk.apple.capacity.limiter.api.Classification; +import io.servicetalk.context.api.ContextMap; +import io.servicetalk.http.api.StreamingHttpRequest; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import static io.servicetalk.apple.traffic.resilience.http.NoOpTrafficResiliencyObserver.NO_OP_TICKET_OBSERVER; + +final class SafeTrafficResiliencyObserver implements TrafficResiliencyObserver { + + private final TrafficResiliencyObserver original; + private static final Logger LOGGER = LoggerFactory.getLogger(SafeTrafficResiliencyObserver.class); + + SafeTrafficResiliencyObserver(final TrafficResiliencyObserver original) { + this.original = original; + } + + @Override + public void onRejectedUnmatchedPartition(final StreamingHttpRequest request) { + try { + this.original.onRejectedUnmatchedPartition(request); + } catch (Throwable t) { + LOGGER.error("Error during onRejectedUnmatchedPartition", t); + } + } + + @Override + public void onRejectedLimit(final StreamingHttpRequest request, final String limiter, + final ContextMap meta, final Classification classification) { + try { + this.original.onRejectedLimit(request, limiter, meta, classification); + } catch (Throwable t) { + LOGGER.error("Error during onRejectedLimit", t); + } + } + + @Override + public void onRejectedOpenCircuit(final StreamingHttpRequest request, final String breaker, + final ContextMap meta, final Classification classification) { + try { + this.original.onRejectedOpenCircuit(request, breaker, meta, classification); + } catch (Throwable t) { + LOGGER.error("Error during onRejectedOpenCircuit", t); + } + } + + @Nullable + @Override + public TicketObserver onAllowedThrough(final StreamingHttpRequest request, + @Nullable final CapacityLimiter.LimiterState state) { + try { + return this.original.onAllowedThrough(request, state); + } catch (Throwable t) { + LOGGER.error("Error during onAllowedThrough", t); + return NO_OP_TICKET_OBSERVER; + } + } +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/StateContext.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/StateContext.java new file mode 100644 index 0000000000..b4430e7c53 --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/StateContext.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.circuit.breaker.api.CircuitBreaker; + +import javax.annotation.Nullable; + +/** + * State information of the {@link TrafficResilienceHttpServiceFilter traffic-resilience} service filter. + */ +public final class StateContext { + + @Nullable + private final CircuitBreaker breaker; + + StateContext(@Nullable final CircuitBreaker breaker) { + this.breaker = breaker; + } + + /** + * Returns the {@link CircuitBreaker} in-use for the currently evaluated request. + * @return The {@link CircuitBreaker} in-use for the currently evaluated request. + */ + @Nullable + public CircuitBreaker breaker() { + return breaker; + } +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrackPendingRequestsHttpFilter.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrackPendingRequestsHttpFilter.java new file mode 100644 index 0000000000..867c9a9673 --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrackPendingRequestsHttpFilter.java @@ -0,0 +1,182 @@ +/* + * Copyright © 2023 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter.Ticket; +import io.servicetalk.concurrent.api.Single; +import io.servicetalk.concurrent.api.TerminalSignalConsumer; +import io.servicetalk.http.api.FilterableStreamingHttpClient; +import io.servicetalk.http.api.HttpExecutionStrategies; +import io.servicetalk.http.api.HttpExecutionStrategy; +import io.servicetalk.http.api.HttpServiceContext; +import io.servicetalk.http.api.StreamingHttpClientFilter; +import io.servicetalk.http.api.StreamingHttpClientFilterFactory; +import io.servicetalk.http.api.StreamingHttpRequest; +import io.servicetalk.http.api.StreamingHttpRequester; +import io.servicetalk.http.api.StreamingHttpResponse; +import io.servicetalk.http.api.StreamingHttpResponseFactory; +import io.servicetalk.http.api.StreamingHttpService; +import io.servicetalk.http.api.StreamingHttpServiceFilter; +import io.servicetalk.http.api.StreamingHttpServiceFilterFactory; +import io.servicetalk.http.utils.BeforeFinallyHttpOperator; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +import static io.servicetalk.concurrent.api.Single.defer; +import static java.util.concurrent.atomic.AtomicIntegerFieldUpdater.newUpdater; + +/** + * A filter that tracks number of pending requests. + * We would like to check the hypothesis if we somehow loose the {@link Ticket} inside + * {@link AbstractTrafficManagementHttpFilter} or if the counter is misaligned for a different reason, like incorrect + * handling of terminal events by {@link BeforeFinallyHttpOperator}. + */ +final class TrackPendingRequestsHttpFilter implements StreamingHttpClientFilterFactory, + StreamingHttpServiceFilterFactory { + + private enum Position { + BEFORE, AFTER + } + + static final TrackPendingRequestsHttpFilter BEFORE = new TrackPendingRequestsHttpFilter(Position.BEFORE); + static final TrackPendingRequestsHttpFilter AFTER = new TrackPendingRequestsHttpFilter(Position.AFTER); + + private final Position position; + + private TrackPendingRequestsHttpFilter(final Position position) { + this.position = position; + } + + @Override + public StreamingHttpClientFilter create(final FilterableStreamingHttpClient client) { + return new TrackPendingRequestsHttpClientFilter(client, position); + } + + @Override + public StreamingHttpServiceFilter create(final StreamingHttpService service) { + return new TrackPendingRequestsHttpServiceFilter(service, position); + } + + @Override + public HttpExecutionStrategy requiredOffloads() { + return HttpExecutionStrategies.offloadNone(); + } + + private static final class TrackPendingRequestsHttpClientFilter extends StreamingHttpClientFilter { + + private static final AtomicIntegerFieldUpdater pendingUpdater = + newUpdater(TrackPendingRequestsHttpClientFilter.class, "pending"); + + private volatile int pending; + private final Position position; + + TrackPendingRequestsHttpClientFilter(final FilterableStreamingHttpClient client, final Position position) { + super(client); + this.position = position; + } + + @Override + protected Single request(final StreamingHttpRequester delegate, + final StreamingHttpRequest request) { + return defer(() -> { + pendingUpdater.incrementAndGet(this); + return delegate.request(request) + .liftSync(new BeforeFinallyHttpOperator(new TerminalSignalConsumer() { + @Override + public void onComplete() { + decrement(); + } + + @Override + public void onError(final Throwable throwable) { + decrement(); + } + + @Override + public void cancel() { + decrement(); + } + + private void decrement() { + pendingUpdater.decrementAndGet(TrackPendingRequestsHttpClientFilter.this); + } + }, true)) + .shareContextOnSubscribe(); + }); + } + + @Override + public String toString() { + return getClass().getSimpleName() + + "{pending=" + pending + + ", position=" + position + + '}'; + } + } + + private static final class TrackPendingRequestsHttpServiceFilter extends StreamingHttpServiceFilter { + + private static final AtomicIntegerFieldUpdater pendingUpdater = + newUpdater(TrackPendingRequestsHttpServiceFilter.class, "pending"); + + private volatile int pending; + private final Position position; + + TrackPendingRequestsHttpServiceFilter(final StreamingHttpService service, final Position position) { + super(service); + this.position = position; + } + + @Override + public Single handle(final HttpServiceContext ctx, + final StreamingHttpRequest request, + final StreamingHttpResponseFactory responseFactory) { + return defer(() -> { + pendingUpdater.incrementAndGet(this); + return delegate().handle(ctx, request, responseFactory) + .liftSync(new BeforeFinallyHttpOperator(new TerminalSignalConsumer() { + @Override + public void onComplete() { + decrement(); + } + + @Override + public void onError(final Throwable throwable) { + decrement(); + } + + @Override + public void cancel() { + decrement(); + } + + private void decrement() { + pendingUpdater.decrementAndGet(TrackPendingRequestsHttpServiceFilter.this); + } + }, true)) + .shareContextOnSubscribe(); + }); + } + + @Override + public String toString() { + return getClass().getSimpleName() + + "{pending=" + pending + + ", position=" + position + + '}'; + } + } +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpClientFilter.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpClientFilter.java new file mode 100644 index 0000000000..e084f5e4d6 --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpClientFilter.java @@ -0,0 +1,555 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter; +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter.Ticket; +import io.servicetalk.apple.capacity.limiter.api.Classification; +import io.servicetalk.apple.capacity.limiter.api.RequestRejectedException; +import io.servicetalk.apple.circuit.breaker.api.CircuitBreaker; +import io.servicetalk.apple.traffic.resilience.http.PeerCapacityRejectionPolicy.PassthroughRequestRejectedException; +import io.servicetalk.concurrent.api.Executor; +import io.servicetalk.concurrent.api.Single; +import io.servicetalk.http.api.FilterableStreamingHttpClient; +import io.servicetalk.http.api.HttpClient; +import io.servicetalk.http.api.HttpRequestMetaData; +import io.servicetalk.http.api.HttpResponseMetaData; +import io.servicetalk.http.api.HttpResponseStatus; +import io.servicetalk.http.api.StreamingHttpClientFilter; +import io.servicetalk.http.api.StreamingHttpClientFilterFactory; +import io.servicetalk.http.api.StreamingHttpRequest; +import io.servicetalk.http.api.StreamingHttpRequester; +import io.servicetalk.http.api.StreamingHttpResponse; +import io.servicetalk.http.api.StreamingHttpResponseFactory; +import io.servicetalk.http.netty.RetryingHttpRequesterFilter; +import io.servicetalk.http.utils.TimeoutHttpRequesterFilter; +import io.servicetalk.transport.api.ServerListenContext; + +import java.time.Duration; +import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; +import javax.annotation.Nullable; + +import static io.servicetalk.apple.traffic.resilience.http.PeerCapacityRejectionPolicy.Type.REJECT; +import static io.servicetalk.apple.traffic.resilience.http.PeerCapacityRejectionPolicy.Type.REJECT_PASSTHROUGH; +import static io.servicetalk.apple.traffic.resilience.http.PeerCapacityRejectionPolicy.Type.REJECT_RETRY; +import static io.servicetalk.concurrent.internal.ThrowableUtils.unknownStackTrace; +import static io.servicetalk.http.api.HttpResponseStatus.BAD_GATEWAY; +import static io.servicetalk.http.api.HttpResponseStatus.SERVICE_UNAVAILABLE; +import static io.servicetalk.http.api.HttpResponseStatus.TOO_MANY_REQUESTS; +import static io.servicetalk.utils.internal.DurationUtils.isPositive; +import static java.lang.Integer.MAX_VALUE; +import static java.time.Duration.ZERO; +import static java.util.Objects.requireNonNull; + +/** + * A {@link StreamingHttpClientFilterFactory} to enforce capacity and circuit-breaking control for a client. + * Requests that are not able to acquire a capacity ticket or a circuit permit, + * will fail with a {@link RequestRejectedException}. + *

+ *

Ordering of filters

+ * Ordering of the {@link TrafficResilienceHttpClientFilter capacity-filter} is important for various reasons: + *
    + *
  • The traffic control filter should be as early as possible in the execution chain to offer a fast-fail + * reaction and ideally trigger a natural back-pressure mechanism with the transport.
    • + *
  • The traffic control filter should not be offloaded if possible to avoid situations where + * continuous traffic overflows the offloading subsystem.
    • + *
  • The traffic control filter should be ordered after a + * {@link RetryingHttpRequesterFilter retry-filter} if one is used, to avail the + * benefit of retrying requests that failed due to (local or remote) capacity issues. + * {@link RetryableRequestRejectedException} are safely retry-able errors, since they occur on the outgoing + * side before they even touch the network. {@link DelayedRetryRequestRejectedException} errors on the other + * side, are remote rejections, and its up to the application logic to opt-in for them to be retryable, by + * configuring the relevant predicate of the {@link RetryingHttpRequesterFilter retry + * -filter}
    • + *
  • The traffic control filter should be ordered after a + * {@link RetryingHttpRequesterFilter retry-filter} to allow an already acquired + * {@link Ticket permit} to be released in case + * of other errors/timeouts of the operation, before retrying to re-acquire a + * {@link Ticket permit}. Otherwise, a + * {@link Ticket permit} may be held idle for as + * long as the operation is awaiting to be re-tried, thus, mis-utilising available resources for other requests + * through the same {@link HttpClient client}. + *
    • + *
  • If the traffic control filter is ordered after a + * {@link TimeoutHttpRequesterFilter timeout-filter} then a potential timeout will be + * delivered to it in the form of a cancellation. The default terminal callback for the ticket in that case, is + * set to {@link Ticket#dropped() dropped} to avail for local throttling, since a timeout is a good indicator + * that a sub-process in the pipeline is not completing fast enough. + *
    • + *
  • If the traffic control filter is ordered before a + * {@link TimeoutHttpRequesterFilter timeout-filter} then a potential timeout will be + * delivered to it in the form of a {@link TimeoutException}, which is in turn triggers the + * {@link Ticket#dropped() drop-event of the ticket} by default. Behavior can be overridden through this + * {@link Builder#onErrorTicketTerminal(BiConsumer)}. + *
    • + *
+ * + */ +public final class TrafficResilienceHttpClientFilter extends AbstractTrafficManagementHttpFilter + implements StreamingHttpClientFilterFactory { + + private static final RequestRejectedException LOCAL_REJECTION_RETRYABLE_EXCEPTION = unknownStackTrace( + new RetryableRequestRejectedException("Local capacity rejection", null, false, true), + TrafficResilienceHttpClientFilter.class, "localRejection"); + + private static final Single RETRYABLE_LOCAL_CAPACITY_REJECTION = + Single.failed(LOCAL_REJECTION_RETRYABLE_EXCEPTION); + + /** + * Default rejection observer for dropped requests from an external sourced. + * see. {@link Builder#peerCapacityRejection(PeerCapacityRejectionPolicy)}. + * + * The default predicate matches the following HTTP response codes: + *
    + *
  • {@link HttpResponseStatus#TOO_MANY_REQUESTS}
    • + *
  • {@link HttpResponseStatus#BAD_GATEWAY}
    • + *
  • {@link HttpResponseStatus#SERVICE_UNAVAILABLE}
    • + *
+ *

+ * If a {@link CircuitBreaker} is used consider adjusting this predicate to avoid considering + * {@link HttpResponseStatus#SERVICE_UNAVAILABLE} as a capacity issue. + */ + public static final Predicate DEFAULT_CAPACITY_REJECTION_PREDICATE = metaData -> + // Some proxies are known to return BAD_GATEWAY when the upstream is unresponsive (i.e. heavy load). + metaData.status().code() == TOO_MANY_REQUESTS.code() || metaData.status().code() == BAD_GATEWAY.code() || + metaData.status().code() == SERVICE_UNAVAILABLE.code(); + + /** + * Default rejection observer for dropped requests from an external sourced due to service unavailability. + * see. {@link Builder#peerBreakerRejection(HttpResponseMetaData, CircuitBreaker)}}. + * + * The default predicate matches the following HTTP response codes: + *

    + *
  • {@link HttpResponseStatus#SERVICE_UNAVAILABLE}
    • + *
+ */ + public static final Predicate DEFAULT_BREAKER_REJECTION_PREDICATE = metaData -> + metaData.status().code() == SERVICE_UNAVAILABLE.code(); + + private final PeerCapacityRejectionPolicy peerCapacityRejectionPolicy; + private final boolean forceOpenCircuitOnPeerCircuitRejections; + @Nullable + private final Function focreOpenCircuitOnPeerCircuitRejectionsDelayProvider; + @Nullable + private final Executor circuitBreakerResetExecutor; + + private TrafficResilienceHttpClientFilter(final Supplier> + capacityPartitionsSupplier, + final boolean rejectWhenNotMatchedCapacityPartition, + final Supplier> + circuitBreakerPartitionsSupplier, + final Function classifier, + final PeerCapacityRejectionPolicy peerCapacityRejectionPolicy, + final Predicate breakerRejectionPredicate, + final Consumer onCompletion, + final Consumer onCancellation, + final BiConsumer onError, + final boolean forceOpenCircuitOnPeerCircuitRejections, + @Nullable final Function + focreOpenCircuitOnPeerCircuitRejectionsDelayProvider, + @Nullable final Executor circuitBreakerResetExecutor, + final TrafficResiliencyObserver observer) { + super(capacityPartitionsSupplier, rejectWhenNotMatchedCapacityPartition, classifier, + peerCapacityRejectionPolicy.predicate(), breakerRejectionPredicate, onCompletion, onCancellation, + onError, circuitBreakerPartitionsSupplier, observer); + this.peerCapacityRejectionPolicy = peerCapacityRejectionPolicy; + this.forceOpenCircuitOnPeerCircuitRejections = forceOpenCircuitOnPeerCircuitRejections; + this.focreOpenCircuitOnPeerCircuitRejectionsDelayProvider = + focreOpenCircuitOnPeerCircuitRejectionsDelayProvider; + this.circuitBreakerResetExecutor = circuitBreakerResetExecutor; + } + + @Override + public StreamingHttpClientFilter create(final FilterableStreamingHttpClient client) { + return TrackPendingRequestsHttpFilter.BEFORE.create(new StreamingHttpClientFilter( + TrackPendingRequestsHttpFilter.AFTER.create(client)) { + + final Function capacityPartitions = newCapacityPartitions(); + final Function circuitBreakerPartitions = + newCircuitBreakerPartitions(); + + @Override + protected Single request(final StreamingHttpRequester delegate, + final StreamingHttpRequest request) { + return applyCapacityControl(capacityPartitions, circuitBreakerPartitions, + null, request, null, delegate::request) + .onErrorResume(PassthroughRequestRejectedException.class, t -> Single.succeeded(t.response())); + } + }); + } + + @Override + protected Single handleLocalBreakerRejection( + StreamingHttpRequest request, @Nullable StreamingHttpResponseFactory responseFactory, + @Nullable CircuitBreaker breaker) { + return DEFAULT_BREAKER_REJECTION; + } + + @Override + protected Single handleLocalCapacityRejection( + @Nullable final ServerListenContext serverListenContext, + StreamingHttpRequest request, @Nullable StreamingHttpResponseFactory responseFactory) { + return RETRYABLE_LOCAL_CAPACITY_REJECTION; + } + + @Override + RuntimeException peerCapacityRejection(final StreamingHttpResponse resp) { + final PeerCapacityRejectionPolicy.Type type = peerCapacityRejectionPolicy.type(); + if (type == REJECT_RETRY) { + final Duration delay = peerCapacityRejectionPolicy.delayProvider().apply(resp); + return new DelayedRetryRequestRejectedException(delay); + } else if (type == REJECT) { + return super.peerCapacityRejection(resp); + } else if (type == REJECT_PASSTHROUGH) { + return new PassthroughRequestRejectedException("Service under heavy load", resp); + } else { + return new IllegalStateException("Unexpected PeerCapacityRejectionPolicy.Type: " + type); + } + } + + @Override + RuntimeException peerBreakerRejection(final HttpResponseMetaData resp, final CircuitBreaker breaker) { + if (forceOpenCircuitOnPeerCircuitRejections) { + assert focreOpenCircuitOnPeerCircuitRejectionsDelayProvider != null; + assert circuitBreakerResetExecutor != null; + final Duration delay = focreOpenCircuitOnPeerCircuitRejectionsDelayProvider.apply(resp); + if (isPositive(delay)) { + breaker.forceOpenState(); + circuitBreakerResetExecutor.schedule(breaker::reset, delay); + } + } + + return super.peerBreakerRejection(resp, breaker); + } + + /** + * A {@link TrafficResilienceHttpServiceFilter} instance builder. + */ + public static final class Builder { + private Supplier> capacityPartitionsSupplier; + private boolean rejectWhenNotMatchedCapacityPartition; + private Supplier> circuitBreakerPartitionsSupplier = + () -> __ -> null; + private Function classifier = __ -> () -> MAX_VALUE; + private PeerCapacityRejectionPolicy peerCapacityRejectionPolicy = + new PeerCapacityRejectionPolicy(DEFAULT_CAPACITY_REJECTION_PREDICATE, REJECT_RETRY, __ -> ZERO); + private Predicate peerUnavailableRejectionPredicate = DEFAULT_BREAKER_REJECTION_PREDICATE; + private final Consumer onCompletionTicketTerminal = Ticket::completed; + private Consumer onCancellationTicketTerminal = Ticket::dropped; + private BiConsumer onErrorTicketTerminal = (ticket, throwable) -> { + if (throwable instanceof RequestRejectedException || throwable instanceof TimeoutException) { + ticket.dropped(); + } else { + ticket.failed(throwable); + } + }; + private boolean forceOpenCircuitOnPeerCircuitRejections; + @Nullable + private Function focreOpenCircuitOnPeerCircuitRejectionsDelayProvider; + @Nullable + private Executor circuitBreakerResetExecutor; + private TrafficResiliencyObserver observer = NoOpTrafficResiliencyObserver.INSTANCE; + + /** + * A {@link TrafficResilienceHttpClientFilter} with no partitioning schemes. + *

+ * All requests will go through the {@link CapacityLimiter}. + * + * @param capacityLimiterSupplier The {@link Supplier} to create a new {@link CapacityLimiter} for each new + * filter created by this {@link StreamingHttpClientFilterFactory factory}. + */ + public Builder(Supplier capacityLimiterSupplier) { + requireNonNull(capacityLimiterSupplier); + this.capacityPartitionsSupplier = () -> { + CapacityLimiter capacityLimiter = capacityLimiterSupplier.get(); + return __ -> capacityLimiter; + }; + this.rejectWhenNotMatchedCapacityPartition = true; + } + + /** + * A {@link TrafficResilienceHttpClientFilter} can support request partitioning schemes. + *

+ * A partition in the context of capacity management, is a set of requests that represent an application + * characteristic relevant to capacity, which can be isolated and have their own set of rules + * (ie. {@link CapacityLimiter}). + *

+ * An example of a partition can be to represent each customer in a multi-tenant service. + * If an application wants to introduce customer API quotas, they can do so by identifying that customer + * through the {@link HttpRequestMetaData} and providing a different {@link CapacityLimiter} for that customer. + *

+ * If a {@code partitions} doesn't return a {@link CapacityLimiter} for the given {@link HttpRequestMetaData} + * then the {@code rejectNotMatched} is evaluated to decide what the filter should do with this request. + * If {@code true} then the request will be {@link RequestRejectedException rejected}. + *

+ * It's important that instances returned from this {@link Function mapper} are singletons and shared + * across the same matched partitions. Otherwise, capacity will not be controlled as expected, and there + * is also the risk for {@link OutOfMemoryError}. + * + * @param capacityPartitionsSupplier A {@link Supplier} to create a new {@link Function} for each new filter + * created by this {@link StreamingHttpClientFilterFactory factory}. + * Function provides a {@link CapacityLimiter} instance for the given {@link HttpRequestMetaData}. + * @param rejectNotMatched Flag that decides what the filter should do when {@code partitions} doesn't return + * a {@link CapacityLimiter}. + */ + public Builder(final Supplier> capacityPartitionsSupplier, + final boolean rejectNotMatched) { + this.capacityPartitionsSupplier = requireNonNull(capacityPartitionsSupplier); + this.rejectWhenNotMatchedCapacityPartition = rejectNotMatched; + } + + /** + * Define {@link CapacityLimiter} partitions. + *

+ * A partition in the context of capacity management, is a set of requests that represent an application + * characteristic relevant to capacity, which can be isolated and have their own set of rules + * (ie. {@link CapacityLimiter}). + *

+ * An example of a partition can be to represent each customer in a multi-tenant service. + * If an application wants to introduce customer API quotas, they can do so by identifying that customer + * through the {@link HttpRequestMetaData} and providing a different {@link CapacityLimiter} for that customer. + *

+ * If a {@code partitions} doesn't return a {@link CapacityLimiter} for the given {@link HttpRequestMetaData} + * then the {@code rejectNotMatched} is evaluated to decide what the filter should do with this request. + * If {@code true} then the request will be {@link RequestRejectedException rejected}. + *

+ * It's important that instances returned from this {@link Function mapper} are singletons and shared + * across the same matched partitions. Otherwise, capacity will not be controlled as expected, and there + * is also the risk for {@link OutOfMemoryError}. + * + * @param capacityPartitionsSupplier A {@link Supplier} to create a new {@link Function} for each new filter + * created by this {@link StreamingHttpClientFilterFactory factory}. + * Function provides a {@link CapacityLimiter} instance for the given {@link HttpRequestMetaData}. + * @param rejectNotMatched Flag that decides what the filter should do when {@code partitions} doesn't return + * a {@link CapacityLimiter}. + * @return {@code this} + */ + public Builder capacityPartitions( + final Supplier> capacityPartitionsSupplier, + final boolean rejectNotMatched) { + this.capacityPartitionsSupplier = requireNonNull(capacityPartitionsSupplier); + this.rejectWhenNotMatchedCapacityPartition = rejectNotMatched; + return this; + } + + /** + * Classification in the context of capacity management allows for hints to the relevant + * {@link CapacityLimiter} to be influenced on the decision-making process by the class of the + * {@link HttpRequestMetaData request}. + *

+ * An example of classification, could be health checks that need to be given preference and still allowed + * a permit even under stress conditions. Another case, could be a separation of reads and writes, giving + * preference to the reads will result in a more available system under stress, by rejecting earlier writes. + *

+ * The behavior of the classification and their thresholds could be different among different + * {@link CapacityLimiter} implementations, therefore the use of this API requires good understanding of how + * the algorithm in use will react for the different classifications. + *

+ * Classification works within the context of a single + * {@link #Builder(Supplier, boolean)} partition} and not universally in the filter. + *

+ * It's worth noting that classification is strictly a hint and could be ignored by the + * {@link CapacityLimiter}. + * @param classifier A {@link Function} that maps an incoming {@link HttpRequestMetaData} to a + * {@link Classification}. + * @return {@code this}. + */ + public Builder classifier(final Function classifier) { + this.classifier = requireNonNull(classifier); + return this; + } + + /** + * Define {@link CircuitBreaker} to manage local or remote errors. + *

+ * The breakers can either be universal or follow any partitioning scheme (i.e., API / service-path, customer + * etc) but is recommended to follow similar schematics between service and client if possible for best + * experience. + *

+ * The matching {@link CircuitBreaker} for a {@link HttpRequestMetaData request} can be forced opened due to + * a remote open circuit-breaker (i.e., {@link HttpResponseStatus#SERVICE_UNAVAILABLE}) + * dissallowing further outgoing requests for a fixed periods; + * {@link #forceOpenCircuitOnPeerCircuitRejections(Function, Executor)}. + * + * @param circuitBreakerPartitionsSupplier A {@link Supplier} to create a new {@link Function} for each new + * filter created by this {@link StreamingHttpClientFilterFactory factory}. + * Function provides a {@link CircuitBreaker} instance for the given {@link HttpRequestMetaData}. + * @return {@code this}. + */ + public Builder circuitBreakerPartitions( + final Supplier> circuitBreakerPartitionsSupplier) { + this.circuitBreakerPartitionsSupplier = requireNonNull(circuitBreakerPartitionsSupplier); + return this; + } + + /** + * Peers can reject and exception due to capacity reasons based on their own principals and implementation + * details. A {@link TrafficResilienceHttpClientFilter} can benefit from this input as feedback for the + * {@link CapacityLimiter} in use, that the request was dropped (ie. rejected), thus it can also bring its + * local limit down to help with the overloaded peer. Since what defines a rejection/drop or request for + * backpressure is not universally common, one can define what response characteristics define that state. + *

+ * It's important to know that if the passed {@code rejectionPredicate} tests {@code true} for a given + * {@link HttpResponseMetaData} then the operation is {@link Single#failed(Throwable)}. + *

+ * Out of the box if nothing custom is defined, the filter recognises as rejections requests with the following + * status codes: + *

    + *
  • {@link HttpResponseStatus#TOO_MANY_REQUESTS}
    • + *
  • {@link HttpResponseStatus#BAD_GATEWAY}
    • + *
+ * + *

+ * Allowing retry, requests will fail with a {@link DelayedRetryRequestRejectedException} to support + * retrying mechanisms (like retry-filters or retry operators) to re-attempt the same request. + * Requests that fail due to capacity limitation, are good candidates for a retry, since we anticipate they are + * safe to be executed again (no previous invocation actually started) and because this maximizes the success + * chances. + * @param policy The {@link PeerCapacityRejectionPolicy} that represents the peer capacity rejection behavior. + * @return {@code this}. + */ + public Builder peerCapacityRejection(final PeerCapacityRejectionPolicy policy) { + this.peerCapacityRejectionPolicy = requireNonNull(policy); + return this; + } + + /** + * Peers can reject requests due to service unavailability. + * A {@link TrafficResilienceHttpClientFilter} can benefit from this input as feedback for the + * {@link CircuitBreaker} in use. A similar exception can be generated locally as a result of that feedback, + * to help the active local {@link CircuitBreaker} to also adapt. + *

+ * It's important to know that if the passed {@code rejectionPredicate} tests {@code true} for a given + * {@link HttpResponseMetaData} then the operation will be {@link Single#failed(Throwable)}. + *

+ * Out of the box if nothing custom is defined, the filter recognises as rejections requests with the following + * status codes: + *

    + *
  • {@link HttpResponseStatus#SERVICE_UNAVAILABLE}
    • + *
+ * + * @param rejectionPredicate The {@link Function} that resolves a {@link HttpResponseMetaData response} to a + * peer-rejection or not. + * @return {@code this}. + */ + public Builder peerUnavailableRejectionPredicate( + final Predicate rejectionPredicate) { + this.peerUnavailableRejectionPredicate = requireNonNull(rejectionPredicate); + return this; + } + + /** + * When a peer rejects a {@link HttpRequestMetaData request} due to an open-circuit (see. + * {@link #peerUnavailableRejectionPredicate(Predicate)}), the feedback can be used + * to also forcefully open the local {@link HttpRequestMetaData request's} {@link CircuitBreaker}. + * The local {@link CircuitBreaker} will close again once a delay period passes as defined/extracted through + * the {@code delayProvider}. + *

+ * If the delay provided is not a positive value, then the {@link CircuitBreaker} will not be modified. + * + * @param delayProvider A function to provide / extract a delay in milliseconds for the + * {@link CircuitBreaker} to remain open. + * @param executor A {@link Executor} used to re-close the {@link CircuitBreaker} once the delay expires. + * @return {@code this}. + */ + public Builder forceOpenCircuitOnPeerCircuitRejections( + final Function delayProvider, + final Executor executor) { + this.forceOpenCircuitOnPeerCircuitRejections = true; + this.focreOpenCircuitOnPeerCircuitRejectionsDelayProvider = requireNonNull(delayProvider); + this.circuitBreakerResetExecutor = requireNonNull(executor); + return this; + } + + /** + * When a peer rejects a {@link HttpRequestMetaData request} due to an open-circuit (see. + * {@link #peerUnavailableRejectionPredicate(Predicate)}), ignore feedback and leave local matching + * {@link CircuitBreaker circuit-breake partition} closed. + * + * @return {@code this}. + */ + public Builder dontForceOpenCircuitOnPeerCircuitRejections() { + this.forceOpenCircuitOnPeerCircuitRejections = false; + this.focreOpenCircuitOnPeerCircuitRejectionsDelayProvider = null; + this.circuitBreakerResetExecutor = null; + return this; + } + + /** + * {@link Ticket Ticket} terminal callback override upon erroneous completion of the request operation. + * Erroneous completion in this context means, that an error occurred as part of the operation or the + * {@link #peerCapacityRejection(PeerCapacityRejectionPolicy)} triggered an exception. + * By default the terminal callback is {@link Ticket#failed(Throwable)}. + * + * @param onError Callback to override default {@link Ticket ticket} terminal event for an erroneous + * operation. + * @return {@code this}. + */ + public Builder onErrorTicketTerminal(final BiConsumer onError) { + this.onErrorTicketTerminal = requireNonNull(onError); + return this; + } + + /** + * {@link Ticket Ticket} terminal callback override upon cancellation of the request operation. + * By default the terminal callback is {@link Ticket#dropped()}. + *

+ * You may need to adjust this callback depending on the ordering this filter was applied. + * For example if the filter is applied after the + * {@link TimeoutHttpRequesterFilter timeout-filter} then you may want to also + * {@link Ticket#dropped() drop the ticket} to let the algorithm apply throttling accounting for this timeout. + * @param onCancellation Callback to override default {@link Ticket ticket} terminal event when an operation + * is cancelled. + * @return {@code this}. + */ + public Builder onCancelTicketTerminal(final Consumer onCancellation) { + this.onCancellationTicketTerminal = requireNonNull(onCancellation); + return this; + } + + /** + * Provide an observer to track interactions of the filter and requests. + * @param observer an observer to track interactions of the filter and requests. + * @return {@code this}. + */ + public Builder observer(final TrafficResiliencyObserver observer) { + requireNonNull(observer); + this.observer = new SafeTrafficResiliencyObserver(observer); + return this; + } + + /** + * Invoke to build an instance of {@link TrafficResilienceHttpClientFilter} filter to be used inside the + * HttpClientBuilder. + * + * @return An instance of {@link TrafficResilienceHttpClientFilter} with the characteristics + * of this builder input. + */ + public TrafficResilienceHttpClientFilter build() { + return new TrafficResilienceHttpClientFilter(capacityPartitionsSupplier, + rejectWhenNotMatchedCapacityPartition, + circuitBreakerPartitionsSupplier, classifier, peerCapacityRejectionPolicy, + peerUnavailableRejectionPredicate, onCompletionTicketTerminal, onCancellationTicketTerminal, + onErrorTicketTerminal, forceOpenCircuitOnPeerCircuitRejections, + focreOpenCircuitOnPeerCircuitRejectionsDelayProvider, circuitBreakerResetExecutor, observer); + } + } +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilter.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilter.java new file mode 100644 index 0000000000..16c29538fa --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilter.java @@ -0,0 +1,688 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter; +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter.Ticket; +import io.servicetalk.apple.capacity.limiter.api.Classification; +import io.servicetalk.apple.capacity.limiter.api.RequestRejectedException; +import io.servicetalk.apple.circuit.breaker.api.CircuitBreaker; +import io.servicetalk.concurrent.api.Single; +import io.servicetalk.http.api.HttpHeaderNames; +import io.servicetalk.http.api.HttpRequestMetaData; +import io.servicetalk.http.api.HttpResponseMetaData; +import io.servicetalk.http.api.HttpServerBuilder; +import io.servicetalk.http.api.HttpServiceContext; +import io.servicetalk.http.api.StreamingHttpRequest; +import io.servicetalk.http.api.StreamingHttpResponse; +import io.servicetalk.http.api.StreamingHttpResponseFactory; +import io.servicetalk.http.api.StreamingHttpService; +import io.servicetalk.http.api.StreamingHttpServiceFilter; +import io.servicetalk.http.api.StreamingHttpServiceFilterFactory; +import io.servicetalk.http.utils.TimeoutHttpServiceFilter; +import io.servicetalk.transport.api.ServerListenContext; + +import java.time.Duration; +import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import javax.annotation.Nullable; + +import static io.servicetalk.buffer.api.CharSequences.newAsciiString; +import static io.servicetalk.concurrent.api.Single.succeeded; +import static io.servicetalk.http.api.HttpHeaderNames.RETRY_AFTER; +import static java.lang.Integer.MAX_VALUE; +import static java.lang.String.valueOf; +import static java.util.Objects.requireNonNull; + +/** + * A {@link StreamingHttpServiceFilterFactory} to enforce capacity control for a server. + * Requests that are not able to acquire a {@link Ticket permit}, will fail with a {@link RequestRejectedException}. + *

+ *

Ordering of filters

+ * Ordering of the {@link TrafficResilienceHttpClientFilter capacity-filter} is important for various reasons: + *
    + *
  • The traffic control filter should be as early as possible in the execution chain to offer a fast-fail + * reaction and ideally trigger a natural back-pressure mechanism with the transport. It's recommended to + * not offload this filter by using + * the {@link HttpServerBuilder#appendNonOffloadingServiceFilter(StreamingHttpServiceFilterFactory)} variant + * when appending to the service builder. Therefore, it's expected that any function provided through + * the {@link Builder}, should not block, to avoid any impact on the I/O threads. since this + * filter will not be offloaded.
    • + *
  • The traffic control filter should not be offloaded if possible to avoid situations where + * continuous traffic overflows the offloading subsystem.
    • + *
  • If the traffic control filter is ordered after a + * {@link TimeoutHttpServiceFilter timeout-filter} then a potential timeout will be + * delivered to it in the form of a cancellation, in which case you may want to override the default + * {@link Builder#onCancellationTicketTerminal terminal event} of the ticket, to {@link Ticket#dropped() drop it} + * to avail for local throttling, since a timeout is a good indicator that a sub-process in the pipeline is not + * completing fast enough. + *
    • + *
  • If the traffic control filter is ordered before a + * {@link TimeoutHttpServiceFilter timeout-filter} then a potential timeout will be + * delivered to it in the form of a {@link TimeoutException}, which is in turn triggers the + * {@link Ticket#dropped() drop-event of the ticket} by default. Behavior can be overridden through this + * {@link Builder#onErrorTicketTerminal callback}. + *
    • + *
+ * + */ +public final class TrafficResilienceHttpServiceFilter extends AbstractTrafficManagementHttpFilter + implements StreamingHttpServiceFilterFactory { + + private final RejectionPolicy rejectionPolicy; + + private TrafficResilienceHttpServiceFilter(final Supplier> + capacityPartitionsSupplier, + final boolean rejectNotMatched, + final Function classifier, + final Consumer onCompletion, + final Consumer onCancellation, + final BiConsumer onError, + final Supplier> + circuitBreakerPartitionsSupplier, + final RejectionPolicy onRejectionPolicy, + final TrafficResiliencyObserver observer) { + super(capacityPartitionsSupplier, rejectNotMatched, classifier, __ -> false, __ -> false, + onCompletion, onCancellation, onError, circuitBreakerPartitionsSupplier, observer); + this.rejectionPolicy = onRejectionPolicy; + } + + @Override + public StreamingHttpServiceFilter create(final StreamingHttpService service) { + return TrackPendingRequestsHttpFilter.BEFORE.create(new StreamingHttpServiceFilter( + TrackPendingRequestsHttpFilter.AFTER.create(service)) { + + final Function capacityPartitions = newCapacityPartitions(); + final Function circuitBreakerPartitions = + newCircuitBreakerPartitions(); + + @Override + public Single handle(final HttpServiceContext ctx, + final StreamingHttpRequest request, + final StreamingHttpResponseFactory responseFactory) { + final ServerListenContext actualContext = ctx.parent() instanceof ServerListenContext ? + (ServerListenContext) ctx.parent() : + ctx; + return applyCapacityControl(capacityPartitions, circuitBreakerPartitions, actualContext, request, + responseFactory, request1 -> delegate().handle(ctx, request1, responseFactory)); + } + }); + } + + @Override + protected Ticket wrapTicket(@Nullable final ServerListenContext serverListenContext, final Ticket ticket) { + return serverListenContext == null ? ticket : new ServerResumptionTicketWrapper(serverListenContext, ticket); + } + + @Override + protected Single handleLocalCapacityRejection( + @Nullable final ServerListenContext serverListenContext, + final StreamingHttpRequest request, + @Nullable final StreamingHttpResponseFactory responseFactory) { + assert serverListenContext != null; + if (rejectionPolicy.onLimitStopAcceptingConnections) { + serverListenContext.acceptConnections(false); + } + + if (responseFactory != null) { + return rejectionPolicy.onLimitResponseBuilder + .apply(request, responseFactory) + .map(resp -> { + rejectionPolicy.onLimitRetryAfter.accept(resp); + return resp; + }); + } + + return DEFAULT_CAPACITY_REJECTION; + } + + @Override + protected Single handleLocalBreakerRejection( + final StreamingHttpRequest request, + @Nullable final StreamingHttpResponseFactory responseFactory, + @Nullable final CircuitBreaker breaker) { + if (responseFactory != null) { + return rejectionPolicy.onOpenCircuitResponseBuilder + .apply(request, responseFactory) + .map(resp -> { + rejectionPolicy.onOpenCircuitRetryAfter + .accept(resp, new StateContext(breaker)); + return resp; + }) + .shareContextOnSubscribe(); + } + + return DEFAULT_BREAKER_REJECTION; + } + + /** + * Default response rejection policy. + *
    + *
  • When a request is rejected due to capacity, the service will respond + * {@link RejectionPolicy#tooManyRequests()}.
    • + *
  • When a request is rejected due to capacity, the service will NOT include a retry-after header.
    • + *
  • When a request is rejected due to breaker, the service will respond + * {@link RejectionPolicy#serviceUnavailable()}.
    • + *
  • When a request is rejected due to breaker, the service will respond with Retry-After header hinting + * the duration the breaker will remain open.
    • + *
+ * + * @return The default {@link RejectionPolicy}. + */ + public static RejectionPolicy defaultRejectionResponsePolicy() { + return new RejectionPolicy.Builder().build(); + } + + /** + * A {@link TrafficResilienceHttpServiceFilter} instance builder. + * + */ + public static final class Builder { + private boolean rejectNotMatched; + private Supplier> capacityPartitionsSupplier; + private Function classifier = __ -> () -> MAX_VALUE; + private Supplier> circuitBreakerPartitionsSupplier = + () -> __ -> null; + private RejectionPolicy onRejectionPolicy = defaultRejectionResponsePolicy(); + private final Consumer onCompletionTicketTerminal = Ticket::completed; + private Consumer onCancellationTicketTerminal = Ticket::ignored; + private BiConsumer onErrorTicketTerminal = (ticket, throwable) -> { + if (throwable instanceof RequestRejectedException || throwable instanceof TimeoutException) { + ticket.dropped(); + } else { + ticket.failed(throwable); + } + }; + private TrafficResiliencyObserver observer = NoOpTrafficResiliencyObserver.INSTANCE; + + /** + * A {@link TrafficResilienceHttpServiceFilter} with no partitioning schemes. + *

+ * All requests will go through the provided {@link CapacityLimiter}. + * + * @param capacityLimiterSupplier The {@link Supplier} to create a new {@link CapacityLimiter} for each new + * filter created by this {@link StreamingHttpServiceFilterFactory factory}. + */ + public Builder(Supplier capacityLimiterSupplier) { + requireNonNull(capacityLimiterSupplier, "capacityLimiterSupplier"); + this.capacityPartitionsSupplier = () -> { + CapacityLimiter capacityLimiter = capacityLimiterSupplier.get(); + return __ -> capacityLimiter; + }; + this.rejectNotMatched = true; + } + + /** + * A {@link TrafficResilienceHttpServiceFilter} can support request partitioning schemes. + *

+ * A partition in the context of capacity management, is a set of requests that represent an application + * characteristic relevant to capacity, which can be isolated and have their own set of rules + * (ie. {@link CapacityLimiter}). + *

+ * An example of a partition can be to represent each customer in a multi-tenant service. + * If an application wants to introduce customer API quotas, they can do so by identifying that customer + * through the {@link HttpRequestMetaData} and providing a different {@link CapacityLimiter} for that customer. + *

+ * If a {@code partitions} doesn't return a {@link CapacityLimiter} for the given {@link HttpRequestMetaData} + * then the {@code rejectNotMatched} is evaluated to decide what the filter should do with this request. + * If {@code true} then the request will be {@link RequestRejectedException rejected}. + *

+ * It's important that instances returned from this {@link Function mapper} are singletons and shared + * across the same matched partitions. Otherwise, capacity will not be controlled as expected, and there + * is also the risk for {@link OutOfMemoryError}. + * + * @param capacityPartitionsSupplier A {@link Supplier} to create a new {@link Function} for each new filter + * created by this {@link StreamingHttpServiceFilterFactory factory}. + * Function provides a {@link CapacityLimiter} instance for the given {@link HttpRequestMetaData}. + * @param rejectNotMatched Flag that decides what the filter should do when {@code partitions} doesn't return + * a {@link CapacityLimiter}. + */ + public Builder(final Supplier> capacityPartitionsSupplier, + final boolean rejectNotMatched) { + this.capacityPartitionsSupplier = requireNonNull(capacityPartitionsSupplier, "capacityPartitionsSupplier"); + this.rejectNotMatched = rejectNotMatched; + } + + /** + * Define {@link CapacityLimiter} partitions. + *

+ * A partition in the context of capacity management, is a set of requests that represent an application + * characteristic relevant to capacity, which can be isolated and have their own set of rules + * (i.e. {@link CapacityLimiter}). + *

+ * An example of a partition can be to represent each customer in a multi-tenant service. + * If an application wants to introduce customer API quotas, they can do so by identifying that customer + * through the {@link HttpRequestMetaData} and providing a different {@link CapacityLimiter} for that customer. + *

+ * If a {@code partitions} doesn't return a {@link CapacityLimiter} for the given {@link HttpRequestMetaData} + * then the {@code rejectNotMatched} is evaluated to decide what the filter should do with this request. + * If {@code true} then the request will be {@link RequestRejectedException rejected}. + *

+ * It's important that instances returned from this {@link Function mapper} are singletons and shared + * across the same matched partitions. Otherwise, capacity will not be controlled as expected, and there + * is also the risk for {@link OutOfMemoryError}. + * + * @param capacityPartitionsSupplier A {@link Supplier} to create a new {@link Function} for each new filter + * created by this {@link StreamingHttpServiceFilterFactory factory}. + * Function provides a {@link CapacityLimiter} instance for the given {@link HttpRequestMetaData}. + * @param rejectNotMatched Flag that decides what the filter should do when {@code partitions} doesn't return + * a {@link CapacityLimiter}. + * @return {@code this}. + */ + public Builder capacityPartitions( + final Supplier> capacityPartitionsSupplier, + final boolean rejectNotMatched) { + this.capacityPartitionsSupplier = requireNonNull(capacityPartitionsSupplier, "capacityPartitionsSupplier"); + this.rejectNotMatched = rejectNotMatched; + return this; + } + + /** + * Classification in the context of capacity management allows for hints to the relevant + * {@link CapacityLimiter} to be influenced on the decision making process by the class of the + * {@link HttpRequestMetaData request}. + *

+ * An example of classification, could be health checks that need to be given preference and still allowed + * a permit even under stress conditions. Another case, could be a separation of reads and writes, giving + * preference to the reads will result in a more available system under stress, by rejecting earlier writes. + *

+ * The behavior of the classification and their thresholds could be different among different + * {@link CapacityLimiter} implementations, therefore the use of this API requires good understanding of how + * the algorithm in use will react for the different classifications. + *

+ * Classification work within the context of a single {@link #Builder(Supplier, boolean)} partition} + * and not universally in the filter. + * @param classifier A {@link Function} that maps an incoming {@link HttpRequestMetaData} to a + * {@link Classification}. + * @return {@code this}. + */ + public Builder classifier(final Function classifier) { + this.classifier = requireNonNull(classifier, "classifier"); + return this; + } + + /** + * Define {@link CircuitBreaker} partitions to manage local errors. + *

+ * The breakers can either be universal or follow any partitioning scheme (i.e., API / service-path, customer + * e.t.c) but is recommended to follow similar schematics between service and client if possible for best + * experience. + *

+ * Once a matching {@link CircuitBreaker} transitions to open state, requests that match the same breaker + * will fail (e.g., {@link io.servicetalk.http.api.HttpResponseStatus#SERVICE_UNAVAILABLE}) and + * {@link RejectionPolicy#onOpenCircuitRetryAfter} can be used to hint peers about the fact that + * the circuit will remain open for a certain amount of time. + * + * @param circuitBreakerPartitionsSupplier A {@link Supplier} to create a new {@link Function} for each new + * filter created by this {@link StreamingHttpServiceFilterFactory factory}. + * Function provides a {@link CircuitBreaker} instance for the given {@link HttpRequestMetaData}. + * @return {@code this}. + */ + public Builder circuitBreakerPartitions( + final Supplier> circuitBreakerPartitionsSupplier) { + this.circuitBreakerPartitionsSupplier = requireNonNull(circuitBreakerPartitionsSupplier, + "circuitBreakerPartitionsSupplier"); + return this; + } + + /** + * {@link Ticket Ticket} terminal callback override upon erroneous completion of the operation. + * Erroneous completion in this context means, that an error occurred for either the {@link Single} or the + * {@link io.servicetalk.concurrent.api.Publisher} of the operation. + * By default, the terminal callback is {@link Ticket#failed(Throwable)}. + * + * @param onError Callback to override default {@link Ticket ticket} terminal event for an erroneous + * operation. + * @return {@code this}. + */ + public Builder onErrorTicketTerminal(final BiConsumer onError) { + this.onErrorTicketTerminal = requireNonNull(onError, "onError"); + return this; + } + + /** + * {@link Ticket Ticket} terminal callback override upon cancellation of the operation. + * By default, the terminal callback is {@link Ticket#ignored()}. + *

+ * You may need to adjust this callback depending on the ordering this filter was applied. + * For example if the filter is applied after the + * {@link io.servicetalk.http.utils.TimeoutHttpRequesterFilter timeout-filter} then you may want to also + * {@link Ticket#dropped() drop the ticket} to let the algorithm apply throttling accounting for this timeout. + * @param onCancellation Callback to override default {@link Ticket ticket} terminal event when an operation + * is cancelled. + * @return {@code this}. + */ + public Builder onCancelTicketTerminal(final Consumer onCancellation) { + this.onCancellationTicketTerminal = requireNonNull(onCancellation, "onCancellation"); + return this; + } + + /** + * Defines the {@link RejectionPolicy} which in turn defines the behavior of the service when a + * rejection occurs due to {@link CapacityLimiter capacity} or {@link CircuitBreaker breaker}. + * + * @param policy The policy to put into effect when a rejection occurs. + * @return {@code this}. + */ + public Builder onRejectionPolicy(final RejectionPolicy policy) { + this.onRejectionPolicy = requireNonNull(policy, "policy"); + return this; + } + + /** + * Provide an observer to track interactions of the filter and requests. + * @param observer an observer to track interactions of the filter and requests. + * @return {@code this}. + */ + public Builder observer(final TrafficResiliencyObserver observer) { + requireNonNull(observer, "observer"); + this.observer = new SafeTrafficResiliencyObserver(observer); + return this; + } + + /** + * Invoke to build an instance of {@link TrafficResilienceHttpServiceFilter} filter to be used inside the + * {@link HttpServerBuilder}. + * @return An instance of {@link TrafficResilienceHttpServiceFilter} with the characteristics + * of this builder input. + */ + public TrafficResilienceHttpServiceFilter build() { + return new TrafficResilienceHttpServiceFilter(capacityPartitionsSupplier, rejectNotMatched, + classifier, onCompletionTicketTerminal, onCancellationTicketTerminal, + onErrorTicketTerminal, circuitBreakerPartitionsSupplier, onRejectionPolicy, observer); + } + } + + /** + * Policy to rule the behavior of service rejections due to capacity or open circuit. + */ + public static final class RejectionPolicy { + + /** + * Custom retry-after header that supports milliseconds resolution, rather than seconds. + */ + public static final CharSequence RETRY_AFTER_MILLIS = newAsciiString("retry-after-millis"); + + private final BiFunction> + onLimitResponseBuilder; + + private final Consumer onLimitRetryAfter; + + private final boolean onLimitStopAcceptingConnections; + + private final BiFunction> + onOpenCircuitResponseBuilder; + + private final BiConsumer onOpenCircuitRetryAfter; + + private RejectionPolicy(final BiFunction> onLimitResponseBuilder, + final Consumer onLimitRetryAfter, + final boolean onLimitStopAcceptingConnections, + final BiFunction> onOpenCircuitResponseBuilder, + final BiConsumer + onOpenCircuitRetryAfter) { + this.onLimitResponseBuilder = onLimitResponseBuilder; + this.onLimitRetryAfter = onLimitRetryAfter; + this.onLimitStopAcceptingConnections = onLimitStopAcceptingConnections; + this.onOpenCircuitResponseBuilder = onOpenCircuitResponseBuilder; + this.onOpenCircuitRetryAfter = onOpenCircuitRetryAfter; + } + + /** + * A hard-coded delay in seconds to be supplied as a Retry-After HTTP header in a {@link HttpResponseMetaData}. + * + * @param seconds The value (in seconds) to be used in the Retry-After header. + * @return A {@link HttpResponseMetaData} consumer, that enhances the headers with a fixed Retry-After figure in + * seconds. + */ + public static Consumer retryAfterHint(final int seconds) { + final CharSequence secondsSeq = newAsciiString(valueOf(seconds)); + return resp -> resp.addHeader(RETRY_AFTER, secondsSeq); + } + + /** + * A delay in seconds to be supplied as a Retry-After HTTP header in a {@link HttpResponseMetaData} based on the + * {@link CircuitBreaker} that matched the {@link HttpRequestMetaData}. + * + * @param fallbackSeconds The value (in seconds) to be used if no {@link CircuitBreaker} matched. + * @return A {@link HttpResponseMetaData} consumer, that enhances the headers with a Retry-After figure in + * seconds based on the duration the matching {@link CircuitBreaker} will remain open, or a fallback period. + */ + public static BiConsumer + retryAfterHintOfBreaker(final int fallbackSeconds) { + final CharSequence secondsSeq = newAsciiString(valueOf(fallbackSeconds)); + return (resp, state) -> { + if (state.breaker() != null || fallbackSeconds > 0) { + resp.setHeader(RETRY_AFTER, state.breaker() != null ? newAsciiString(valueOf( + state.breaker().remainingDurationInOpenState().getSeconds())) : secondsSeq); + } + }; + } + + /** + * A hard-coded delay in milliseconds to be supplied as a Retry-After-Millis HTTP header in a + * {@link HttpResponseMetaData}. Being a custom Http header, it will require special handling on the peer side. + * + * @param duration The duration to be used in the Retry-After-Millis header. + * @return A {@link HttpResponseMetaData} consumer, that enhances the headers with a fixed + * Retry-After-Millis figure in milliseconds. + */ + public static BiConsumer retryAfterMillisHint(final Duration duration) { + final CharSequence millisSeq = newAsciiString(valueOf(duration.toMillis())); + return (resp, breaker) -> resp.setHeader(RETRY_AFTER_MILLIS, millisSeq); + } + + /** + * Pre-defined {@link StreamingHttpResponse response} that signals + * {@link io.servicetalk.http.api.HttpResponseStatus#TOO_MANY_REQUESTS} to the peer. + * + * @return A {@link BiFunction} that regardless the input, it will always return a + * {@link StreamingHttpResponseFactory#tooManyRequests() too-many-requests} response. + */ + public static BiFunction> + tooManyRequests() { + return (__, factory) -> succeeded(factory.tooManyRequests()); + } + + /** + * Pre-defined {@link StreamingHttpResponse response} that signals + * {@link io.servicetalk.http.api.HttpResponseStatus#SERVICE_UNAVAILABLE} to the peer. + * + * @return A {@link BiFunction} that regardless the input, it will always return a + * {@link StreamingHttpResponseFactory#serviceUnavailable() service-unavailable} response. + */ + public static BiFunction> + serviceUnavailable() { + return (__, factory) -> succeeded(factory.serviceUnavailable()); + } + + /** + * A {@link RejectionPolicy} builder to support a custom policy. + */ + public static final class Builder { + private BiFunction> + onLimitResponseBuilder = tooManyRequests(); + + private Consumer onLimitRetryAfter = __ -> { }; + + private boolean onLimitStopAcceptingConnections; + + private BiFunction> + onOpenCircuitResponseBuilder = serviceUnavailable(); + + private BiConsumer onOpenCircuitRetryAfter = + retryAfterHintOfBreaker(-1); + + /** + * Determines the {@link StreamingHttpResponse} when a capacity limit is met. + * + * @param onLimitResponseBuilder A factory function used to generate a {@link StreamingHttpResponse} based + * on the {@link HttpRequestMetaData request} when a {@link CapacityLimiter capacity} limit is observed. + * @return {@code this}. + */ + public Builder onLimitResponseBuilder(final BiFunction> onLimitResponseBuilder) { + this.onLimitResponseBuilder = requireNonNull(onLimitResponseBuilder); + return this; + } + + /** + * Determines a {@link HttpHeaderNames#RETRY_AFTER retry-after} header in the + * {@link StreamingHttpResponse} when a capacity limit is met. + * + * @param onLimitRetryAfter A {@link HttpResponseMetaData} consumer, that can allow response decoration with + * additional headers to hint the peer (upon capacity limits) about a possible wait-time before a + * retry could be issued. + * @return {@code this}. + */ + public Builder onLimitRetryAfter(final Consumer onLimitRetryAfter) { + this.onLimitRetryAfter = requireNonNull(onLimitRetryAfter); + return this; + } + + /** + * When a certain {@link CapacityLimiter} rejects a request due to the active limit, + * (e.g., no {@link Ticket} is returned) influence the server to also stop accepting new connections + * until the capacity is under healthy conditions again. + * This setting only works when a {@link CapacityLimiter} matches the incoming request, in cases this + * doesn't hold (see. {@link TrafficResilienceHttpServiceFilter.Builder#Builder(Supplier, boolean)} + * Builder's rejectedNotMatched argument}) this won't be effective. + *

+ * When a server socket stops accepting new connections + * (see. {@link HttpServiceContext#acceptConnections(boolean)}) due to capacity concerns, the state will be + * toggled back when the {@link Ticket ticket's} terminal callback ({@link Ticket#dropped() dropped}, + * {@link Ticket#failed(Throwable) failed}, {@link Ticket#completed() completed}, {@link Ticket#ignored() + * ignored}) returns a positive or negative value, demonstrating available capacity or not_supported + * respectively. When the returned value is {@code 0} that means no-capacity available, which will keep the + * server in the not-accepting mode. + *

+ * When enabling this feature, it's recommended for clients using this service to configure timeouts + * for their opening connection time and connection idleness time. For example, a client without + * connection-timeout or idle-timeout on the outgoing connections towards this service, won't be able to + * detect on time the connection delays. Likewise, on the server side you can configure the + * {@link io.servicetalk.transport.api.ServiceTalkSocketOptions#SO_BACKLOG server backlog} to a very small + * number or even disable it completely, to avoid holding established connections in the OS. + *

+ * Worth noting that established connections that stay in the OS backlog, usually have a First In First Out + * behavior, which depending on the size of that queue, may result in extending latencies on newer + * requests because older ones are served first. Disabling the + * {@link io.servicetalk.transport.api.ServiceTalkSocketOptions#SO_BACKLOG server backlog} will give a + * better behavior. + * @param stopAccepting {@code true} will allow this filter to control the connection acceptance of the + * overall server socket. + * @return {@code this}. + */ + public Builder onLimitStopAcceptingConnections(final boolean stopAccepting) { + this.onLimitStopAcceptingConnections = stopAccepting; + return this; + } + + /** + * Determines the {@link StreamingHttpResponse} when a circuit-breaker limit is met. + * + * @param onOpenCircuitResponseBuilder A factory function used to generate a {@link StreamingHttpResponse} + * based on the {@link HttpRequestMetaData request} when an open {@link CircuitBreaker breaker} is observed. + * @return {@code this}. + */ + public Builder onOpenCircuitResponseBuilder(final BiFunction> onOpenCircuitResponseBuilder) { + this.onOpenCircuitResponseBuilder = requireNonNull(onOpenCircuitResponseBuilder); + return this; + } + + /** + * Determines a {@link HttpHeaderNames#RETRY_AFTER retry-after} header in the + * {@link StreamingHttpResponse} when a capacity limit is met. + * + * @param onOpenCircuitRetryAfter A {@link HttpResponseMetaData} consumer, that can allow response + * decoration with additional headers to hint the peer (upon open breaker) about a possible wait-time + * before a retry could be issued. + * @return {@code this}. + */ + public Builder onOpenCircuitRetryAfter(final BiConsumer onOpenCircuitRetryAfter) { + this.onOpenCircuitRetryAfter = requireNonNull(onOpenCircuitRetryAfter); + return this; + } + + /** + * Return a custom {@link RejectionPolicy} based on the options of this builder. + * @return A custom {@link RejectionPolicy} based on the options of this builder. + */ + public RejectionPolicy build() { + return new RejectionPolicy(onLimitResponseBuilder, onLimitRetryAfter, onLimitStopAcceptingConnections, + onOpenCircuitResponseBuilder, onOpenCircuitRetryAfter); + } + } + } + + private static final class ServerResumptionTicketWrapper implements Ticket { + private final Ticket ticket; + private final ServerListenContext listenContext; + + private ServerResumptionTicketWrapper(final ServerListenContext listenContext, final Ticket ticket) { + this.ticket = ticket; + this.listenContext = listenContext; + } + + @Override + public CapacityLimiter.LimiterState state() { + return ticket.state(); + } + + @Override + public int completed() { + final int result = ticket.completed(); + if (result == -1 || result > 0) { + listenContext.acceptConnections(true); + } + return result; + } + + @Override + public int dropped() { + final int result = ticket.dropped(); + if (result == -1 || result > 0) { + listenContext.acceptConnections(true); + } + return result; + } + + @Override + public int failed(final Throwable error) { + final int result = ticket.failed(error); + if (result == -1 || result > 0) { + listenContext.acceptConnections(true); + } + return result; + } + + @Override + public int ignored() { + final int result = ticket.ignored(); + if (result == -1 || result > 0) { + listenContext.acceptConnections(true); + } + return result; + } + } +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResiliencyObserver.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResiliencyObserver.java new file mode 100644 index 0000000000..7598315546 --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/TrafficResiliencyObserver.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter; +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter.LimiterState; +import io.servicetalk.apple.capacity.limiter.api.Classification; +import io.servicetalk.apple.circuit.breaker.api.CircuitBreaker; +import io.servicetalk.context.api.ContextMap; +import io.servicetalk.http.api.StreamingHttpRequest; + +import javax.annotation.Nullable; + +/** + * A {@link TrafficResilienceHttpServiceFilter} or {@link TrafficResilienceHttpClientFilter} observer. + * Tracks interactions with {@link CapacityLimiter}s and/or {@link CircuitBreaker}s, and exposes a transactional + * {@link TicketObserver} for each request that was let-through. + *

+ * Note: All interactions with callbacks in the file below, are executed within the flow + * of each request, and are expected to be **non-blocking**. Any blocking calls within the implementation may impact + * negatively the performance of your application. + * + */ +public interface TrafficResiliencyObserver { + + /** + * Transactional observer for the requests that were let-through. + * Allows the caller to track the result of the ticket. + */ + interface TicketObserver { + /** + * Called when the request was completed successfully. + */ + void onComplete(); + + /** + * Called when the request flow was cancelled. + */ + void onCancel(); + + /** + * Called when the request flow terminated erroneously. + * @param throwable the {@link Throwable} that caused the request to fail. + */ + void onError(Throwable throwable); + } + + /** + * Called when a request was "soft-rejected" due to unmatched partition. + * Note: The decision of whether the request was let through or rejected depends on the configuration of the filter. + * @param request the {@link StreamingHttpRequest} correlating to this rejection. + */ + void onRejectedUnmatchedPartition(StreamingHttpRequest request); + + /** + * Called when a request was "hard-rejected" due to a {@link CapacityLimiter} reaching its limit. + * + * @param request the {@link StreamingHttpRequest} correlates to this rejection. + * @param capacityLimiter the {@link CapacityLimiter}'s name that correlates to this traffic flow. + * @param meta the {@link ContextMap} that correlates to this request (if any). + * @param classification the {@link Classification} that correlates to this request (if any). + */ + void onRejectedLimit(StreamingHttpRequest request, String capacityLimiter, ContextMap meta, + Classification classification); + + /** + * Called when a request was "hard-rejected" due to a {@link CircuitBreaker} open state. + * + * @param request the {@link StreamingHttpRequest} correlates to this rejection. + * @param circuitBreaker the {@link CircuitBreaker}'s name that correlates to this traffic flow. + * @param meta the {@link ContextMap} that correlates to this request (if any). + * @param classification the {@link Classification} that correlates to this request (if any). + */ + void onRejectedOpenCircuit(StreamingHttpRequest request, String circuitBreaker, + ContextMap meta, Classification classification); + + /** + * Called when a request was let through. + * + * @param request the {@link StreamingHttpRequest} correlates to this rejection. + * @param state the {@link LimiterState} that correlates to this accepted request. + * @return A {@link TicketObserver} to track the state of the allowed request. + */ + TicketObserver onAllowedThrough(StreamingHttpRequest request, @Nullable LimiterState state); +} diff --git a/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/package-info.java b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/package-info.java new file mode 100644 index 0000000000..49fae433cf --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/main/java/io/servicetalk/apple/traffic/resilience/http/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@ElementsAreNonnullByDefault +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.annotations.ElementsAreNonnullByDefault; diff --git a/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/CapacityClientServerTest.java b/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/CapacityClientServerTest.java new file mode 100644 index 0000000000..fc80d02d7f --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/CapacityClientServerTest.java @@ -0,0 +1,155 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter; +import io.servicetalk.apple.capacity.limiter.api.RequestRejectedException; +import io.servicetalk.apple.traffic.resilience.http.TrafficResilienceHttpServiceFilter.RejectionPolicy; +import io.servicetalk.concurrent.api.Single; +import io.servicetalk.concurrent.api.TestSingle; +import io.servicetalk.concurrent.test.internal.TestSingleSubscriber; +import io.servicetalk.http.api.HttpClient; +import io.servicetalk.http.api.HttpResponse; +import io.servicetalk.http.api.HttpServerBuilder; +import io.servicetalk.http.api.HttpServiceContext; +import io.servicetalk.http.api.SingleAddressHttpClientBuilder; +import io.servicetalk.http.api.StreamingHttpRequest; +import io.servicetalk.http.api.StreamingHttpResponse; +import io.servicetalk.http.api.StreamingHttpResponseFactory; +import io.servicetalk.http.api.StreamingHttpServiceFilter; +import io.servicetalk.transport.api.HostAndPort; +import io.servicetalk.transport.api.ServerContext; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.net.InetSocketAddress; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static io.servicetalk.apple.capacity.limiter.api.CapacityLimiters.fixedCapacity; +import static io.servicetalk.concurrent.api.AsyncCloseables.newCompositeCloseable; +import static io.servicetalk.concurrent.api.SourceAdapters.toSource; +import static io.servicetalk.http.api.HttpResponseStatus.OK; +import static io.servicetalk.http.api.HttpResponseStatus.SERVICE_UNAVAILABLE; +import static io.servicetalk.http.netty.HttpClients.forSingleAddress; +import static io.servicetalk.http.netty.HttpServers.forAddress; +import static io.servicetalk.transport.netty.internal.AddressUtils.localAddress; +import static io.servicetalk.transport.netty.internal.AddressUtils.serverHostAndPort; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class CapacityClientServerTest { + + private final BlockingQueue> serverResponseQueue = new LinkedBlockingQueue<>(); + private ServerContext ctx; + private HttpClient client; + + private void setUp(final boolean applyOnClient, + final Supplier limiterSupplier) + throws Exception { + final HttpServerBuilder serverBuilder = forAddress(localAddress(0)); + serverBuilder.appendServiceFilter(original -> new StreamingHttpServiceFilter(original) { + @Override + public Single handle(final HttpServiceContext ctx, + final StreamingHttpRequest request, + final StreamingHttpResponseFactory responseFactory) { + return delegate().handle(ctx, request, responseFactory); + } + }); + if (!applyOnClient) { + TrafficResilienceHttpServiceFilter serviceFilter = + new TrafficResilienceHttpServiceFilter.Builder(limiterSupplier) + .onRejectionPolicy(new RejectionPolicy.Builder() + .onLimitResponseBuilder(RejectionPolicy.serviceUnavailable()).build()) + .build(); + + serverBuilder.appendServiceFilter(serviceFilter); + } + ctx = serverBuilder.listenAndAwait((__, ___, responseFactory) -> { + TestSingle resp = new TestSingle<>(); + serverResponseQueue.add(resp); + return resp; + }); + final SingleAddressHttpClientBuilder clientBuilder = + forSingleAddress(serverHostAndPort(ctx)); + if (applyOnClient) { + final TrafficResilienceHttpClientFilter clientFilter = + new TrafficResilienceHttpClientFilter.Builder(() -> { + final CapacityLimiter limiter = limiterSupplier.get(); + return __ -> limiter; + }, true).build(); + + clientBuilder.appendClientFilter(clientFilter); + } + client = clientBuilder.build(); + } + + @AfterEach + void tearDown() throws Exception { + newCompositeCloseable().appendAll(client, ctx).close(); + } + + static Stream data() { + return Stream.of(newParam(true, () -> fixedCapacity().capacity(1).build()), + newParam(false, () -> fixedCapacity().capacity(1).build())); + } + + private static Arguments newParam( + final boolean applyOnClient, + final Supplier capacityProvider) { + return Arguments.of(applyOnClient, capacityProvider); + } + + @ParameterizedTest(name = "Apply on client? {0}") + @MethodSource("data") + void underCapacity(final boolean applyOnClient, + final Supplier capacityProvider) throws Exception { + setUp(applyOnClient, capacityProvider); + TestSingleSubscriber responseSub = new TestSingleSubscriber<>(); + CountDownLatch latch = new CountDownLatch(1); + toSource(client.request(client.get("/")).afterFinally(latch::countDown)).subscribe(responseSub); + serverResponseQueue.take().onSuccess(client.httpResponseFactory().ok()); + latch.await(); + final HttpResponse response = responseSub.awaitOnSuccess(); + assertThat("Unexpected result.", response, is(notNullValue())); + assertThat("Unexpected result.", response.status(), is(OK)); + } + + @ParameterizedTest(name = "Apply on client? {0}") + @MethodSource("data") + void overCapacity(final boolean applyOnClient, + final Supplier capacityProvider) throws Exception { + setUp(applyOnClient, capacityProvider); + TestSingleSubscriber response1Sub = new TestSingleSubscriber<>(); + toSource(client.request(client.get("/"))).subscribe(response1Sub); // never completes + serverResponseQueue.take(); // Ensure the request reaches the server. + + if (applyOnClient) { + assertThrows(RequestRejectedException.class, () -> client.asBlockingClient().request(client.get("/"))); + } else { + final HttpResponse response2 = client.asBlockingClient().request(client.get("/")); + assertThat("Unexpected result.", response2.status(), is(SERVICE_UNAVAILABLE)); + } + } +} diff --git a/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpClientFilterTest.java b/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpClientFilterTest.java new file mode 100644 index 0000000000..8465034c44 --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpClientFilterTest.java @@ -0,0 +1,154 @@ +/* + * Copyright © 2022 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter; +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter.Ticket; +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiters; +import io.servicetalk.apple.capacity.limiter.api.RequestRejectedException; +import io.servicetalk.concurrent.api.Publisher; +import io.servicetalk.concurrent.api.Single; +import io.servicetalk.concurrent.api.test.StepVerifiers; +import io.servicetalk.concurrent.internal.DeliberateException; +import io.servicetalk.http.api.DefaultHttpHeadersFactory; +import io.servicetalk.http.api.DefaultStreamingHttpRequestResponseFactory; +import io.servicetalk.http.api.FilterableStreamingHttpClient; +import io.servicetalk.http.api.HttpRequestMethod; +import io.servicetalk.http.api.HttpResponse; +import io.servicetalk.http.api.StreamingHttpClientFilter; +import io.servicetalk.http.api.StreamingHttpRequest; +import io.servicetalk.http.api.StreamingHttpRequestResponseFactory; +import io.servicetalk.http.api.StreamingHttpResponse; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +import static io.servicetalk.apple.traffic.resilience.http.PeerCapacityRejectionPolicy.ofPassthrough; +import static io.servicetalk.apple.traffic.resilience.http.PeerCapacityRejectionPolicy.ofRejection; +import static io.servicetalk.buffer.netty.BufferAllocators.DEFAULT_ALLOCATOR; +import static io.servicetalk.concurrent.internal.DeliberateException.DELIBERATE_EXCEPTION; +import static io.servicetalk.http.api.HttpProtocolVersion.HTTP_1_1; +import static io.servicetalk.http.api.HttpResponseStatus.BAD_GATEWAY; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +class TrafficResilienceHttpClientFilterTest { + private static final StreamingHttpRequestResponseFactory REQ_RES_FACTORY = + new DefaultStreamingHttpRequestResponseFactory(DEFAULT_ALLOCATOR, DefaultHttpHeadersFactory.INSTANCE, + HTTP_1_1); + + private static final StreamingHttpRequest REQUEST = REQ_RES_FACTORY.newRequest(HttpRequestMethod.GET, ""); + + @Test + void verifyPeerRetryableRejection() { + final TrafficResilienceHttpClientFilter trafficResilienceHttpClientFilter = + new TrafficResilienceHttpClientFilter.Builder( + () -> CapacityLimiters.fixedCapacity().capacity(1).build()).build(); + + FilterableStreamingHttpClient client = mock(FilterableStreamingHttpClient.class); + when(client.request(any())).thenReturn(Single.succeeded(REQ_RES_FACTORY.newResponse(BAD_GATEWAY))); + + final StreamingHttpClientFilter clientWithFilter = trafficResilienceHttpClientFilter.create(client); + assertThrows(DelayedRetryRequestRejectedException.class, () -> { + try { + clientWithFilter.request(REQUEST).toFuture().get(); + } catch (ExecutionException e) { + throw e.getCause(); + } + }); + } + + @Test + void verifyPeerRejection() { + final TrafficResilienceHttpClientFilter trafficResilienceHttpClientFilter = + new TrafficResilienceHttpClientFilter.Builder( + () -> CapacityLimiters.fixedCapacity().capacity(1).build()) + .peerCapacityRejection(ofRejection(resp -> BAD_GATEWAY.equals(resp.status()))) + .build(); + + FilterableStreamingHttpClient client = mock(FilterableStreamingHttpClient.class); + AtomicBoolean payloadDrained = new AtomicBoolean(); + when(client.request(any())).thenReturn(Single.succeeded(REQ_RES_FACTORY.newResponse(BAD_GATEWAY) + // Use non-replayable payload body: + .payloadBody(Publisher.fromInputStream(new ByteArrayInputStream("content".getBytes(UTF_8))) + .map(DEFAULT_ALLOCATOR::wrap).whenOnComplete(() -> payloadDrained.set(true))))); + + final StreamingHttpClientFilter clientWithFilter = trafficResilienceHttpClientFilter.create(client); + assertThrows(RequestRejectedException.class, () -> { + try { + clientWithFilter.request(REQUEST).toFuture().get(); + } catch (ExecutionException e) { + throw e.getCause(); + } + }); + assertThat(payloadDrained.get(), is(true)); + } + + @Test + void verifyPeerRejectionPassthrough() throws Exception { + final TrafficResilienceHttpClientFilter trafficResilienceHttpClientFilter = + new TrafficResilienceHttpClientFilter.Builder( + () -> CapacityLimiters.fixedCapacity().capacity(1).build()) + .peerCapacityRejection(ofPassthrough(resp -> BAD_GATEWAY.equals(resp.status()))) + .build(); + + FilterableStreamingHttpClient client = mock(FilterableStreamingHttpClient.class); + when(client.request(any())).thenReturn(Single.succeeded(REQ_RES_FACTORY.newResponse(BAD_GATEWAY) + // Use non-replayable payload body: + .payloadBody(Publisher.fromInputStream(new ByteArrayInputStream("content".getBytes(UTF_8))) + .map(DEFAULT_ALLOCATOR::wrap)))); + + final StreamingHttpClientFilter clientWithFilter = trafficResilienceHttpClientFilter.create(client); + final HttpResponse response = clientWithFilter.request(REQUEST) + .flatMap(StreamingHttpResponse::toResponse).toFuture().get(); + assertThat(response.status(), equalTo(BAD_GATEWAY)); + assertThat(response.payloadBody().toString(UTF_8), is(equalTo("content"))); + } + + @Test + void releaseCapacityIfDelegateThrows() { + CapacityLimiter limiter = mock(CapacityLimiter.class); + Ticket ticket = mock(Ticket.class); + when(limiter.tryAcquire(any(), any())).thenReturn(ticket); + + TrafficResilienceHttpClientFilter filter = + new TrafficResilienceHttpClientFilter.Builder(() -> limiter).build(); + + FilterableStreamingHttpClient client = mock(FilterableStreamingHttpClient.class); + when(client.request(any())).thenThrow(DELIBERATE_EXCEPTION); + + StreamingHttpClientFilter clientWithFilter = filter.create(client); + StepVerifiers.create(clientWithFilter.request(mock(StreamingHttpRequest.class))) + .expectError(DeliberateException.class) + .verify(); + verify(limiter).tryAcquire(any(), any()); + verify(ticket).failed(DELIBERATE_EXCEPTION); + verify(ticket, atLeastOnce()).state(); + verifyNoMoreInteractions(limiter, ticket); + } +} diff --git a/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilterTest.java b/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilterTest.java new file mode 100644 index 0000000000..527f4b7d7f --- /dev/null +++ b/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilterTest.java @@ -0,0 +1,205 @@ +/* + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.apple.traffic.resilience.http; + +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter; +import io.servicetalk.apple.capacity.limiter.api.CapacityLimiters; +import io.servicetalk.apple.traffic.resilience.http.TrafficResilienceHttpServiceFilter.RejectionPolicy; +import io.servicetalk.client.api.ConnectTimeoutException; +import io.servicetalk.concurrent.api.Completable; +import io.servicetalk.concurrent.api.Publisher; +import io.servicetalk.concurrent.api.Single; +import io.servicetalk.concurrent.api.test.StepVerifiers; +import io.servicetalk.concurrent.internal.DeliberateException; +import io.servicetalk.http.api.HttpClient; +import io.servicetalk.http.api.HttpConnection; +import io.servicetalk.http.api.HttpProtocolConfig; +import io.servicetalk.http.api.HttpRequestMethod; +import io.servicetalk.http.api.HttpServerContext; +import io.servicetalk.http.api.HttpServiceContext; +import io.servicetalk.http.api.StreamingHttpClient; +import io.servicetalk.http.api.StreamingHttpRequest; +import io.servicetalk.http.api.StreamingHttpResponseFactory; +import io.servicetalk.http.api.StreamingHttpServiceFilter; +import io.servicetalk.http.netty.HttpClients; +import io.servicetalk.http.netty.HttpServers; +import io.servicetalk.transport.api.ServerContext; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static io.netty.util.internal.PlatformDependent.normalizedOs; +import static io.servicetalk.apple.capacity.limiter.api.CapacityLimiters.fixedCapacity; +import static io.servicetalk.concurrent.api.Single.succeeded; +import static io.servicetalk.concurrent.internal.DeliberateException.DELIBERATE_EXCEPTION; +import static io.servicetalk.http.netty.AsyncContextHttpFilterVerifier.verifyServerFilterAsyncContextVisibility; +import static io.servicetalk.http.netty.HttpProtocolConfigs.h1Default; +import static io.servicetalk.http.netty.HttpProtocolConfigs.h2Default; +import static io.servicetalk.http.netty.HttpServers.forAddress; +import static io.servicetalk.transport.api.ServiceTalkSocketOptions.CONNECT_TIMEOUT; +import static io.servicetalk.transport.api.ServiceTalkSocketOptions.SO_BACKLOG; +import static io.servicetalk.transport.netty.internal.AddressUtils.localAddress; +import static io.servicetalk.transport.netty.internal.AddressUtils.serverHostAndPort; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +class TrafficResilienceHttpServiceFilterTest { + + private static final boolean IS_LINUX = "linux".equals(normalizedOs()); + // There is an off-by-one behavior difference between macOS & Linux. + // Linux has a greater-than check + // (see. https://github.com/torvalds/linux/blob/5bfc75d92efd494db37f5c4c173d3639d4772966/include/net/sock.h#L941) + private static final int TCP_BACKLOG = IS_LINUX ? 0 : 1; + + @Test + void verifyAsyncContext() throws Exception { + verifyServerFilterAsyncContextVisibility( + new TrafficResilienceHttpServiceFilter.Builder(() -> fixedCapacity().capacity(1).build()) + .build()); + } + + @Test + void verifyPeerRejectionCallbacks() throws Exception { + final AtomicInteger consumption = new AtomicInteger(); + // Expect two state changes + final CountDownLatch latch = new CountDownLatch(2); + try (ServerContext serverContext = HttpServers.forPort(0).listenAndAwait((ctx, request, responseFactory) -> + succeeded(responseFactory.serviceUnavailable()))) { + final TrafficResilienceHttpClientFilter trafficResilienceHttpClientFilter = + new TrafficResilienceHttpClientFilter.Builder(() -> CapacityLimiters.fixedCapacity() + .capacity(1) + .stateObserver(consumed -> { + consumption.set(consumed); + latch.countDown(); + }) + .build()).build(); + try (HttpClient httpClient = HttpClients.forSingleAddress(serverHostAndPort(serverContext)) + .appendClientFilter(trafficResilienceHttpClientFilter) + .build()) { + assertThrows(ExecutionException.class, () -> + httpClient.request(httpClient.newRequest(HttpRequestMethod.GET, "/")) + .toFuture().get()); + } finally { + latch.await(); + assertThat("Unexpected limiter consumption", consumption.get(), is(0)); + } + } + } + + @Test + void releaseCapacityIfDelegateThrows() { + CapacityLimiter limiter = mock(CapacityLimiter.class); + CapacityLimiter.Ticket ticket = mock(CapacityLimiter.Ticket.class); + when(limiter.tryAcquire(any(), any())).thenReturn(ticket); + + TrafficResilienceHttpServiceFilter filter = + new TrafficResilienceHttpServiceFilter.Builder(() -> limiter).build(); + + StreamingHttpServiceFilter service = mock(StreamingHttpServiceFilter.class); + when(service.handle(any(), any(), any())).thenThrow(DELIBERATE_EXCEPTION); + + StreamingHttpServiceFilter serviceWithFilter = filter.create(service); + StepVerifiers.create(serviceWithFilter.handle(mock(HttpServiceContext.class), mock(StreamingHttpRequest.class), + mock(StreamingHttpResponseFactory.class))) + .expectError(DeliberateException.class) + .verify(); + verify(limiter).tryAcquire(any(), any()); + verify(ticket).failed(DELIBERATE_EXCEPTION); + verify(ticket, atLeastOnce()).state(); + verifyNoMoreInteractions(limiter, ticket); + } + + enum Protocol { + H1(h1Default()), + H2(h2Default()); + + private final HttpProtocolConfig config; + Protocol(HttpProtocolConfig config) { + this.config = config; + } + } + + @ParameterizedTest + @EnumSource(Protocol.class) + void testStopAcceptingConnections(final Protocol protocol) throws Exception { + final CapacityLimiter limiter = fixedCapacity().capacity(1).build(); + final RejectionPolicy rejectionPolicy = new RejectionPolicy.Builder() + .onLimitStopAcceptingConnections(true) + // Custom response to validate during assertion stage + .onLimitResponseBuilder((meta, respFactory) -> Single.succeeded(respFactory.gatewayTimeout())) + .build(); + TrafficResilienceHttpServiceFilter filter = new TrafficResilienceHttpServiceFilter + .Builder(() -> limiter) + .onRejectionPolicy(rejectionPolicy) + .build(); + + final HttpServerContext serverContext = forAddress(localAddress(0)) + .protocols(protocol.config) + .listenSocketOption(SO_BACKLOG, TCP_BACKLOG) + .appendNonOffloadingServiceFilter(filter) + .listenStreamingAndAwait((ctx, request, responseFactory) -> + succeeded(responseFactory.ok().payloadBody(Publisher.never()))); + + final StreamingHttpClient client = HttpClients.forSingleAddress(serverHostAndPort(serverContext)) + .protocols(protocol.config) + .socketOption(CONNECT_TIMEOUT, (int) SECONDS.toMillis(2)) + .buildStreaming(); + + // First request -> Pending 1 + final StreamingHttpRequest meta1 = client.newRequest(HttpRequestMethod.GET, "/"); + client.reserveConnection(meta1) + .flatMap(it -> it.request(meta1)) + .concat(Completable.defer(() -> { + // First request, has a "never" pub as a body, we don't attempt to consume it. + // Concat second request -> out of capacity -> server yielded + final StreamingHttpRequest meta2 = client.newRequest(HttpRequestMethod.GET, "/"); + return client.reserveConnection(meta2).flatMap(it -> it.request(meta2)).ignoreElement(); + })) + .toFuture() + .get(); + + // Netty will evaluate the "yielding" (i.e., auto-read) on this attempt, so this connection will go through. + assertThat(client.reserveConnection(client.newRequest(HttpRequestMethod.GET, "/")) + .toFuture().get().asConnection(), instanceOf(HttpConnection.class)); + + // This connection shall full-fil the BACKLOG=1 setting + assertThat(client.reserveConnection(client.newRequest(HttpRequestMethod.GET, "/")) + .toFuture().get().asConnection(), instanceOf(HttpConnection.class)); + + // Any attempt to create a connection now, should time out + try { + client.reserveConnection(client.newRequest(HttpRequestMethod.GET, "/")).toFuture().get(); + fail("Expected a connection timeout"); + } catch (ExecutionException e) { + assertThat(e.getCause(), instanceOf(ConnectTimeoutException.class)); + } + } +} diff --git a/settings.gradle b/settings.gradle index 9a6d88cf04..5c09f799d6 100755 --- a/settings.gradle +++ b/settings.gradle @@ -21,6 +21,9 @@ include "servicetalk-annotations", "servicetalk-bom", "servicetalk-buffer-api", "servicetalk-buffer-netty", + "servicetalk-capacity-limiter-api", + "servicetalk-circuit-breaker-api", + "servicetalk-circuit-breaker-resilience4j", "servicetalk-client-api", "servicetalk-client-api-internal", "servicetalk-concurrent", @@ -110,6 +113,7 @@ include "servicetalk-annotations", "servicetalk-serializer-utils", "servicetalk-tcp-netty-internal", "servicetalk-test-resources", + "servicetalk-traffic-resilience-http", "servicetalk-transport-api", "servicetalk-transport-netty", "servicetalk-transport-netty-internal", From 11ce5e5071a9abd84efb3f5cea3c460a7c8e1ecb Mon Sep 17 00:00:00 2001 From: Thomas Kountis Date: Wed, 8 May 2024 07:46:33 -0700 Subject: [PATCH 2/4] downgrade res4j - java8 compat --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 45ad6d8dea..1d1be4c83c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -82,4 +82,4 @@ commonsLangVersion=2.6 grpcVersion=1.61.1 javaxAnnotationsApiVersion=1.3.5 jsonUnitVersion=2.38.0 -resilience4jVersion=2.2.0 +resilience4jVersion=1.7.1 From 8a15df324511e27ff1323f0f685edafd4ed4c735 Mon Sep 17 00:00:00 2001 From: Thomas Kountis Date: Wed, 8 May 2024 11:50:15 -0700 Subject: [PATCH 3/4] comments --- servicetalk-capacity-limiter-api/build.gradle | 3 +- .../api/AimdCapacityLimiterBuilder.java | 19 ++--- .../limiter/api/CompositeCapacityLimiter.java | 2 - .../limiter/api/FixedCapacityLimiter.java | 10 --- .../api/FixedCapacityLimiterBuilder.java | 13 +-- .../limiter/api/GradientCapacityLimiter.java | 19 ----- .../api/GradientCapacityLimiterBuilder.java | 39 +++------ .../capacity/limiter/api/LatencyTracker.java | 9 ++- .../capacity/limiter/api/Preconditions.java | 77 ------------------ ...rafficResilienceHttpServiceFilterTest.java | 2 +- .../utils/internal/NumberUtils.java | 79 +++++++++++++++++++ 11 files changed, 107 insertions(+), 165 deletions(-) delete mode 100644 servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Preconditions.java diff --git a/servicetalk-capacity-limiter-api/build.gradle b/servicetalk-capacity-limiter-api/build.gradle index a268e88f0d..6be6b7fc46 100644 --- a/servicetalk-capacity-limiter-api/build.gradle +++ b/servicetalk-capacity-limiter-api/build.gradle @@ -1,5 +1,5 @@ /* - * Copyright © 2020 Apple Inc. and the ServiceTalk project authors + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ dependencies { implementation project(":servicetalk-annotations") implementation project(":servicetalk-concurrent-internal") + implementation project(":servicetalk-utils-internal") implementation "com.google.code.findbugs:jsr305" implementation "org.slf4j:slf4j-api" diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterBuilder.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterBuilder.java index 7ad4ec79b8..edefe29d38 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterBuilder.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/AimdCapacityLimiterBuilder.java @@ -15,16 +15,17 @@ */ package io.servicetalk.apple.capacity.limiter.api; +import io.servicetalk.utils.internal.DurationUtils; +import io.servicetalk.utils.internal.NumberUtils; + import java.time.Duration; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.LongSupplier; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkBetweenZeroAndOneExclusive; -import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkPositive; -import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkRange; -import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkZeroOrPositive; +import static io.servicetalk.utils.internal.NumberUtils.ensureBetweenZeroAndOneExclusive; +import static io.servicetalk.utils.internal.NumberUtils.ensureRange; import static java.lang.Integer.MAX_VALUE; import static java.util.Objects.requireNonNull; @@ -98,7 +99,7 @@ public AimdCapacityLimiterBuilder limits(final int initial, final int min, final throw new IllegalArgumentException("min: " + min + ", max: " + max + " (expected: min < max)"); } - this.initial = checkRange("initial", initial, min, max); + this.initial = ensureRange(initial, min, max, "initial"); this.min = min; this.max = max; return this; @@ -123,8 +124,8 @@ public AimdCapacityLimiterBuilder limits(final int initial, final int min, final * @return {@code this}. */ public AimdCapacityLimiterBuilder backoffRatio(final float onDrop, final float onLimit) { - this.onDrop = checkBetweenZeroAndOneExclusive("onDrop", onDrop); - this.onLimit = checkBetweenZeroAndOneExclusive("onLimit", onLimit); + this.onDrop = ensureBetweenZeroAndOneExclusive(onDrop, "onDrop"); + this.onLimit = ensureBetweenZeroAndOneExclusive(onLimit, "onLimit"); return this; } @@ -137,7 +138,7 @@ public AimdCapacityLimiterBuilder backoffRatio(final float onDrop, final float o * @return {@code this}. */ public AimdCapacityLimiterBuilder increment(final float increment) { - this.increment = checkPositive("increment", increment); + this.increment = NumberUtils.ensurePositive(increment, "increment"); return this; } @@ -150,7 +151,7 @@ public AimdCapacityLimiterBuilder increment(final float increment) { * @return {@code this}. */ public AimdCapacityLimiterBuilder cooldown(final Duration duration) { - this.cooldown = checkZeroOrPositive("cooldown", duration); + this.cooldown = DurationUtils.ensureNonNegative(duration, "cooldown"); return this; } diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CompositeCapacityLimiter.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CompositeCapacityLimiter.java index 670b76d4ee..548a32427f 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CompositeCapacityLimiter.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CompositeCapacityLimiter.java @@ -28,8 +28,6 @@ * such as overall capacity control along with "specific" (i.e. customer based) partitioned quotas. * The order of the {@link CapacityLimiter} is the same as provided by the user, and the same order is applied * when tickets acquired are released back to their owner. - * - * @param Contextual metadata of the request a {@link CapacityLimiter} supports for evaluation. */ final class CompositeCapacityLimiter implements CapacityLimiter { diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java index 548040edf4..6452a68d76 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java @@ -150,16 +150,6 @@ private static final class CatchAllStateObserver implements StateObserver { this.delegate = delegate; } - @Override - public void observe(final int consumed) { - try { - delegate.observe(consumed); - } catch (Throwable t) { - LOGGER.warn("Unexpected exception from {}.observe({})", - delegate.getClass().getSimpleName(), consumed, t); - } - } - @Override public void observe(final int capacity, final int consumed) { try { diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java index 4f2d3ab1c0..39009cb734 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java @@ -91,23 +91,12 @@ private String name() { */ @FunctionalInterface public interface StateObserver { - /** - * Callback that gives access to internal state of the {@link CapacityLimiter} with fixed capacity. - * - * @param consumed The current consumption (portion of the capacity) of the limiter. - * @deprecated Use {@link #observe(int, int)}. - */ - @Deprecated // FIXME: 0.43 - remove deprecated method or change default impl - void observe(int consumed); - /** * Callback that gives access to internal state of the {@link CapacityLimiter} with fixed capacity. * * @param capacity The max allowed concurrent requests that {@link CapacityLimiter} should allow. * @param consumed The current consumption (portion of the capacity) of the limiter. */ - default void observe(int capacity, int consumed) { - observe(consumed); - } + void observe(int capacity, int consumed); } } diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiter.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiter.java index 0daf9d8d6e..51d9606f99 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiter.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiter.java @@ -129,14 +129,12 @@ public Ticket tryAcquire(final Classification classification, final ContextMap m Ticket ticket = null; synchronized (lock) { newLimit = (int) limit; - newPending = pending; if (pending < limit) { newPending = ++pending; ticket = new DefaultTicket(this, newLimit - newPending); } } - observer.onStateChange(newLimit, newPending); if (ticket != null) { observer.onActiveRequestsIncr(); } @@ -187,10 +185,6 @@ private int onSuccess(final long durationNs) { } } - if (limit > -1) { - observer.onStateChange(limit, newPending); - } - observer.onActiveRequestsDecr(); return limit - newPending; } @@ -205,7 +199,6 @@ private int onDrop() { } observer.onActiveRequestsDecr(); - observer.onStateChange((int) newLimit, newPending); return (int) (newLimit - newPending); } @@ -218,7 +211,6 @@ private int onIgnore() { newPending = --pending; } observer.onActiveRequestsDecr(); - observer.onStateChange((int) newLimit, newPending); return (int) (newLimit - newPending); } @@ -293,17 +285,6 @@ private static final class CatchAllObserver implements Observer { this.delegate = observer; } - @Deprecated - @Override - public void onStateChange(final int limit, final int consumed) { - try { - delegate.onStateChange(limit, consumed); - } catch (Throwable t) { - LOGGER.warn("Unexpected exception from {}.onStateChange({}, {})", - delegate.getClass().getSimpleName(), limit, consumed, t); - } - } - @Override public void onActiveRequestsIncr() { try { diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterBuilder.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterBuilder.java index e7e46df65f..a12c1fd665 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterBuilder.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/GradientCapacityLimiterBuilder.java @@ -15,8 +15,8 @@ */ package io.servicetalk.apple.capacity.limiter.api; -import io.servicetalk.apple.capacity.limiter.api.CapacityLimiter.Ticket; import io.servicetalk.context.api.ContextMap; +import io.servicetalk.utils.internal.NumberUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,10 +44,9 @@ import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.DEFAULT_SUSPEND_INCR; import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.GREEDY_HEADROOM; import static io.servicetalk.apple.capacity.limiter.api.GradientCapacityLimiterProfiles.MIN_SAMPLING_DURATION; -import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkBetweenZeroAndOne; -import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkBetweenZeroAndOneExclusive; -import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkGreaterThan; -import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkPositive; +import static io.servicetalk.utils.internal.NumberUtils.ensureBetweenZeroAndOne; +import static io.servicetalk.utils.internal.NumberUtils.ensureBetweenZeroAndOneExclusive; +import static io.servicetalk.utils.internal.NumberUtils.ensureGreaterThan; import static java.util.Objects.requireNonNull; /** @@ -60,11 +59,6 @@ public final class GradientCapacityLimiterBuilder { private static final AtomicInteger SEQ_GEN = new AtomicInteger(); private static final Observer LOGGING_OBSERVER = new Observer() { - @Override - public void onStateChange(final int limit, final int consumed) { - LOGGER.debug("GradientCapacityLimiter: limit {} consumption {}", limit, consumed); - } - @Override public void onActiveRequestsDecr() { } @@ -131,7 +125,7 @@ public GradientCapacityLimiterBuilder name(final String name) { * @return {@code this}. */ public GradientCapacityLimiterBuilder limits(final int initial, final int min, final int max) { - checkPositive("min", min); + NumberUtils.ensurePositive(min, "min"); if (initial < min || initial > max) { throw new IllegalArgumentException("initial: " + initial + " (expected: min <= initial <= max)"); } @@ -164,8 +158,8 @@ public GradientCapacityLimiterBuilder limits(final int initial, final int min, f * @return {@code this}. */ public GradientCapacityLimiterBuilder backoffRatio(final float onDrop, final float onLimit) { - checkBetweenZeroAndOneExclusive("onDrop", onDrop); - checkBetweenZeroAndOneExclusive("onLimit", onLimit); + ensureBetweenZeroAndOneExclusive(onDrop, "onDrop"); + ensureBetweenZeroAndOneExclusive(onLimit, "onLimit"); this.onDrop = onDrop; this.onLimit = onLimit; @@ -249,7 +243,7 @@ public GradientCapacityLimiterBuilder limitUpdateInterval(final Duration duratio * @return {@code this}. */ public GradientCapacityLimiterBuilder minGradient(final float minGradient) { - checkBetweenZeroAndOne("minGradient", minGradient); + ensureBetweenZeroAndOne(minGradient, "minGradient"); this.minGradient = minGradient; return this; } @@ -264,7 +258,7 @@ public GradientCapacityLimiterBuilder minGradient(final float minGradient) { * @return {@code this}. */ public GradientCapacityLimiterBuilder maxGradient(final float maxPositiveGradient) { - checkGreaterThan("maxGradient", maxPositiveGradient, 1.0f); + ensureGreaterThan(maxPositiveGradient, 1.0f, "maxGradient"); this.maxGradient = maxPositiveGradient; return this; } @@ -342,21 +336,6 @@ private String name() { * A state observer for Gradient {@link CapacityLimiter} to monitor internal state changes. */ public interface Observer { - /** - * Callback that gives access to internal state of the Gradient {@link CapacityLimiter}. - * Useful to capture all consumption changes along with the limit in use, but can be very noisy, - * since consumption changes twice in the lifecycle of a {@link Ticket}. - *

- * The rate of reporting to the observer is based on the rate of change to this - * {@link CapacityLimiter}. - * @param limit The current limit (dynamically computed) of the limiter. - * @param consumed The current consumption (portion of the limit) of the limiter. - * @deprecated alternative for consumed available through {@link #onActiveRequestsIncr} - * and {@link #onActiveRequestsDecr}, similarly alternative for limit changes available through - * {@link #onLimitChange(double, double, double, double, double)}. - */ - @Deprecated - void onStateChange(int limit, int consumed); /** * Callback that informs when the active requests increased by 1. diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/LatencyTracker.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/LatencyTracker.java index af7405a741..035a3b6d49 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/LatencyTracker.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/LatencyTracker.java @@ -1,5 +1,5 @@ /* - * Copyright © 2022 Apple Inc. and the ServiceTalk project authors + * Copyright © 2024 Apple Inc. and the ServiceTalk project authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,17 +15,18 @@ */ package io.servicetalk.apple.capacity.limiter.api; +import io.servicetalk.utils.internal.NumberUtils; + import java.util.function.BiFunction; import javax.annotation.Nullable; -import static io.servicetalk.apple.capacity.limiter.api.Preconditions.checkPositive; import static java.lang.Math.exp; import static java.lang.Math.log; import static java.util.Objects.requireNonNull; /** * A tracker of latency values at certain points in time. - * + *

* This helps observe latency behavior and different implementation can offer their own interpretation of the latency * tracker, allowing a way to enhance the algorithm's behavior according to the observations. * Implementations must provide thread-safety guarantees. @@ -91,7 +92,7 @@ final class EMA implements LatencyTracker { */ EMA(final long halfLifeNs, final LatencyTracker calmTracker, final BiFunction calmRatio) { - checkPositive("halfLifeNs", halfLifeNs); + NumberUtils.ensurePositive(halfLifeNs, "halfLifeNs"); this.tau = halfLifeNs / log(2); this.calmTracker = requireNonNull(calmTracker); this.calmRatio = requireNonNull(calmRatio); diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Preconditions.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Preconditions.java deleted file mode 100644 index 25aabbf556..0000000000 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Preconditions.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright © 2024 Apple Inc. and the ServiceTalk project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.servicetalk.apple.capacity.limiter.api; - -import java.time.Duration; - -import static java.time.Duration.ZERO; - -final class Preconditions { - - private Preconditions() { - // No instances - } - - static int checkPositive(String field, int value) { - if (value <= 0) { - throw new IllegalArgumentException(field + ": " + value + " (expected: > 0)"); - } - return value; - } - - static float checkPositive(String field, float value) { - if (value <= 0.0f) { - throw new IllegalArgumentException(field + ": " + value + " (expected: > 0.0f)"); - } - return value; - } - - static float checkGreaterThan(String field, float value, final float min) { - if (value <= min) { - throw new IllegalArgumentException(field + ": " + value + " (expected: > " + min + ")"); - } - return value; - } - - static float checkBetweenZeroAndOne(String field, float value) { - if (value < 0.0f || value > 1.0f) { - throw new IllegalArgumentException(field + ": " + value + " (expected: 0.0f <= " + field + " <= 1.0f)"); - } - return value; - } - - static int checkRange(String field, int value, int min, int max) { - if (value < min || value > max) { - throw new IllegalArgumentException(field + ": " + value + - " (expected: " + min + " <= " + field + " <= " + max + ")"); - } - return value; - } - - static float checkBetweenZeroAndOneExclusive(String field, float value) { - if (value <= 0.0f || value >= 1.0f) { - throw new IllegalArgumentException(field + ": " + value + " (expected: 0.0f < " + field + " < 1.0f)"); - } - return value; - } - - static Duration checkZeroOrPositive(String field, Duration value) { - if (ZERO.compareTo(value) > 0) { - throw new IllegalArgumentException(field + ": " + value + " (expected: >= " + ZERO + ")"); - } - return value; - } -} diff --git a/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilterTest.java b/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilterTest.java index 527f4b7d7f..4313165f85 100644 --- a/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilterTest.java +++ b/servicetalk-traffic-resilience-http/src/test/java/io/servicetalk/apple/traffic/resilience/http/TrafficResilienceHttpServiceFilterTest.java @@ -96,7 +96,7 @@ void verifyPeerRejectionCallbacks() throws Exception { final TrafficResilienceHttpClientFilter trafficResilienceHttpClientFilter = new TrafficResilienceHttpClientFilter.Builder(() -> CapacityLimiters.fixedCapacity() .capacity(1) - .stateObserver(consumed -> { + .stateObserver((capacity, consumed) -> { consumption.set(consumed); latch.countDown(); }) diff --git a/servicetalk-utils-internal/src/main/java/io/servicetalk/utils/internal/NumberUtils.java b/servicetalk-utils-internal/src/main/java/io/servicetalk/utils/internal/NumberUtils.java index 8a49f24719..27398a3514 100644 --- a/servicetalk-utils-internal/src/main/java/io/servicetalk/utils/internal/NumberUtils.java +++ b/servicetalk-utils-internal/src/main/java/io/servicetalk/utils/internal/NumberUtils.java @@ -39,6 +39,21 @@ public static int ensurePositive(final int value, final String name) { return value; } + /** + * Ensures the float is positive, excluding zero. + * + * @param value the float value to validate + * @param name name of the variable + * @return the passed value if all checks pass + * @throws IllegalArgumentException if the passed float is not greater than zero + */ + public static float ensurePositive(final float value, final String name) { + if (value <= 0.0) { + throw new IllegalArgumentException(name + ": " + value + " (expected > 0.0)"); + } + return value; + } + /** * Ensures the long is positive, excluding zero. * @@ -83,4 +98,68 @@ public static long ensureNonNegative(final long value, final String name) { } return value; } + + /** + * Ensures the float is greater than the min specified. + * + * @param value the float value to validate + * @param min the float min to validate against + * @param field name of the variable + * @return the passed value if all checks pass + * @throws IllegalArgumentException if the passed float doesn't meet the requirements + */ + public static float ensureGreaterThan(final float value, final float min, final String field) { + if (value <= min) { + throw new IllegalArgumentException(field + ": " + value + " (expected: > " + min + ")"); + } + return value; + } + + /** + * Ensures the float is between 0 and 1 (inclusive). + * + * @param value the float value to validate + * @param field name of the variable + * @return the passed value if all checks pass + * @throws IllegalArgumentException if the passed float doesn't meet the requirements + */ + public static float ensureBetweenZeroAndOne(final float value, final String field) { + if (value < 0.0f || value > 1.0f) { + throw new IllegalArgumentException(field + ": " + value + " (expected: 0.0f <= " + field + " <= 1.0f)"); + } + return value; + } + + /** + * Ensures the int is between the provided range (inclusive). + * + * @param value the int value to validate + * @param min the min int value to validate against (inclusive) + * @param max the max int value to validate against (inclusive) + * @param field name of the variable + * @return the passed value if all checks pass + * @throws IllegalArgumentException if the passed int doesn't meet the requirements + */ + public static int ensureRange(final int value, final int min, final int max, final String field) { + if (value < min || value > max) { + throw new IllegalArgumentException(field + ": " + value + + " (expected: " + min + " <= " + field + " <= " + max + ")"); + } + return value; + } + + /** + * Ensures the float is between 0 and 1 (exclusive). + * + * @param value the float value to validate + * @param field name of the variable + * @return the passed value if all checks pass + * @throws IllegalArgumentException if the passed float doesn't meet the requirements + */ + public static float ensureBetweenZeroAndOneExclusive(final float value, final String field) { + if (value <= 0.0f || value >= 1.0f) { + throw new IllegalArgumentException(field + ": " + value + " (expected: 0.0f < " + field + " < 1.0f)"); + } + return value; + } } From c0884beaa313ab95a78e004e78802ed622a6876d Mon Sep 17 00:00:00 2001 From: Thomas Kountis Date: Wed, 8 May 2024 13:24:00 -0700 Subject: [PATCH 4/4] weight -> priority --- .../apple/capacity/limiter/api/CapacityLimiters.java | 8 ++++---- .../apple/capacity/limiter/api/Classification.java | 6 +++--- .../apple/capacity/limiter/api/FixedCapacityLimiter.java | 2 +- .../capacity/limiter/api/FixedCapacityLimiterBuilder.java | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiters.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiters.java index 1b2afb95f5..7e2a9f99bd 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiters.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/CapacityLimiters.java @@ -44,16 +44,16 @@ public static CapacityLimiter composite(final List providers) { * Returns a {@link CapacityLimiter} that will reject all requests till the current pending request count is equal * or less to the passed {@code capacity}. * This {@link CapacityLimiter} takes into consideration the {@link Classification} of a given request and will - * variate the effective {@code capacity} according to the {@link Classification#weight() weight} before + * variate the effective {@code capacity} according to the {@link Classification#priority() weight} before * attempting to grant access to the request. The effective {@code capacity} will never be more than the given * {@code capacity}. *

- * Requests with {@link Classification#weight() weight} equal to or greater than {@code 100} will enjoy - * the full capacity (100%), while requests with {@link Classification#weight() weight} less than {@code 100} + * Requests with {@link Classification#priority() weight} equal to or greater than {@code 100} will enjoy + * the full capacity (100%), while requests with {@link Classification#priority() weight} less than {@code 100} * will be mapped to a percentage point of the given {@code capacity} and be granted access only if the {@code * consumed capacity} is less than that percentage. *
- * Example: With a {@code capacity} = 10, and incoming {@link Classification#weight()} = 70, then the effective + * Example: With a {@code capacity} = 10, and incoming {@link Classification#priority()} = 70, then the effective * target limit for this request will be 70% of the 10 = 7. If current consumption is less than 7, the request * will be permitted. * diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Classification.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Classification.java index 824a3dd78c..afcfe043cb 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Classification.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/Classification.java @@ -32,12 +32,12 @@ @FunctionalInterface public interface Classification { /** - * The weight should be a positive number between 0.1 and 1.0 (inclusive), which hints to a {@link CapacityLimiter} + * The priority should be a positive number between 0 and 100 (inclusive), which hints to a {@link CapacityLimiter} * the importance of a {@link Classification}. * Higher value represents the most important {@link Classification}, while lower value represents less important * {@link Classification}. - * @return A positive value between 0.1 and 1.0 (inclusive) that hints importance of a request to a + * @return A positive value between 0 and 100 (inclusive) that hints importance of a request to a * {@link CapacityLimiter}. */ - int weight(); + int priority(); } diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java index 6452a68d76..766568749f 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiter.java @@ -62,7 +62,7 @@ public String name() { @Override public Ticket tryAcquire(final Classification classification, final ContextMap meta) { - final int weight = min(max(classification.weight(), 0), 100); + final int weight = min(max(classification.priority(), 0), 100); final int effectiveLimit = (capacity * weight) / 100; for (;;) { final int currPending = pending; diff --git a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java index 39009cb734..495d1e1ebe 100644 --- a/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java +++ b/servicetalk-capacity-limiter-api/src/main/java/io/servicetalk/apple/capacity/limiter/api/FixedCapacityLimiterBuilder.java @@ -46,7 +46,7 @@ public FixedCapacityLimiterBuilder name(final String name) { /** * Defines the fixed capacity for the {@link CapacityLimiter}. * Concurrent requests above this figure will be rejected. Requests with particular - * {@link Classification#weight() weight} will be respected and the total capacity for them will be adjusted + * {@link Classification#priority() weight} will be respected and the total capacity for them will be adjusted * accordingly. * @param capacity The max allowed concurrent requests that this {@link CapacityLimiter} should allow. * @return {@code this}.