-
Notifications
You must be signed in to change notification settings - Fork 444
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Rate limit merges by FID (#1213)
* feat: Rate limit merges by FID * changeset * tests
- Loading branch information
1 parent
d675af9
commit 1e0979b
Showing
8 changed files
with
164 additions
and
69 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| "@farcaster/hubble": patch | ||
| --- | ||
|
|
||
| feat: Rate limit merges per FID to the total messages storage available for the FID |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| import { RateLimiterMemory } from "rate-limiter-flexible"; | ||
| import { getRateLimiterForTotalMessages, rateLimitByIp, rateLimitByKey } from "./rateLimits.js"; | ||
| import { sleep } from "./crypto.js"; | ||
|
|
||
| describe("test rate limits", () => { | ||
| const Limit10PerSecond = new RateLimiterMemory({ | ||
| points: 10, | ||
| duration: 1, | ||
| }); | ||
|
|
||
| test("test rate limiting", async () => { | ||
| // 10 Requests should be fine | ||
| for (let i = 0; i < 10; i++) { | ||
| const result = await rateLimitByIp("testip:3000", Limit10PerSecond); | ||
| expect(result.isOk()).toBeTruthy(); | ||
| } | ||
|
|
||
| // Sleep for 1 second to reset the rate limiter | ||
| await sleep(1100); | ||
|
|
||
| // 11th+ request should fail | ||
| for (let i = 0; i < 20; i++) { | ||
| const result = await rateLimitByIp("testip:3000", Limit10PerSecond); | ||
| if (i < 10) { | ||
| expect(result.isOk()).toBeTruthy(); | ||
| } else { | ||
| expect(result._unsafeUnwrapErr().message).toEqual("Too many requests"); | ||
| } | ||
| } | ||
| }); | ||
|
|
||
| test("test dynamic rate limiting", async () => { | ||
| // 10 Requests should be fine for 1st set of messages | ||
| const rateLimiter1 = getRateLimiterForTotalMessages(10, 1); | ||
| const rateLimiter2 = getRateLimiterForTotalMessages(11, 1); | ||
|
|
||
| for (let i = 0; i < 10; i++) { | ||
| const result1 = await rateLimitByKey("3000", rateLimiter1); | ||
| expect(result1.isOk()).toBeTruthy(); | ||
|
|
||
| // same key, but different rate limiter should also be fine | ||
| const result2 = await rateLimitByKey("3000", rateLimiter2); | ||
| expect(result2.isOk()).toBeTruthy(); | ||
| } | ||
|
|
||
| // Sleep for 1 second to reset the rate limiter | ||
| await sleep(1100); | ||
|
|
||
| // 11th+ request should fail | ||
| for (let i = 0; i < 20; i++) { | ||
| const result1 = await rateLimitByKey("3000", rateLimiter1); | ||
| if (i < 10) { | ||
| expect(result1.isOk()).toBeTruthy(); | ||
| } else { | ||
| expect(result1._unsafeUnwrapErr().message).toEqual("Too many requests"); | ||
| } | ||
|
|
||
| // same key, but different rate limiter should pass till the 11th message | ||
| const result2 = await rateLimitByKey("3000", rateLimiter2); | ||
| if (i < 11) { | ||
| expect(result2.isOk()).toBeTruthy(); | ||
| } else { | ||
| expect(result2._unsafeUnwrapErr().message).toEqual("Too many requests"); | ||
| } | ||
| } | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| import { HubAsyncResult, HubError } from "@farcaster/hub-nodejs"; | ||
| import { err, ok } from "neverthrow"; | ||
| import { RateLimiterAbstract, RateLimiterMemory } from "rate-limiter-flexible"; | ||
|
|
||
| // Number of submit messages (total) that can be merged per 60 seconds | ||
| export const SUBMIT_MESSAGE_RATE_LIMIT = { | ||
| points: 20_000, | ||
| duration: 60, | ||
| }; | ||
|
|
||
| // We keep a map of rate limiters per total messages allowed, since each fid has a different limit | ||
| // The totalMessages are always num of storage units purchased * totalPruneSize limit, so there will | ||
| // be as many rate limiters as the number of distinct storage units purchased, which is a small number | ||
| const rateLimiters = new Map<number, RateLimiterMemory>(); | ||
| export function getRateLimiterForTotalMessages(totalMessages: number, duration = 60 * 60 * 24): RateLimiterAbstract { | ||
| if (rateLimiters.has(totalMessages)) { | ||
| return rateLimiters.get(totalMessages) as RateLimiterAbstract; | ||
| } | ||
|
|
||
| const limiter = new RateLimiterMemory({ | ||
| points: totalMessages, | ||
| duration, | ||
| }); | ||
| rateLimiters.set(totalMessages, limiter); | ||
| return limiter; | ||
| } | ||
|
|
||
| /** Rate limit by IP address */ | ||
| export const rateLimitByIp = async (ip: string, limiter: RateLimiterAbstract): HubAsyncResult<boolean> => { | ||
| // Get the IP part of the address | ||
| const ipPart = ip.split(":")[0] ?? ""; | ||
|
|
||
| return rateLimitByKey(ipPart, limiter); | ||
| }; | ||
|
|
||
| /** Rate limit by key for the limiter */ | ||
| export const rateLimitByKey = async (fid: string, limiter: RateLimiterAbstract): HubAsyncResult<boolean> => { | ||
| try { | ||
| await limiter.consume(fid); | ||
| return ok(true); | ||
| } catch (e) { | ||
| return err(new HubError("unavailable", "Too many requests")); | ||
| } | ||
| }; |