Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement rate limit #3454

Merged
merged 11 commits into from
Dec 15, 2021
Prev Previous commit
Next Next commit
Remove requestCountTotalLimit, address PR comments
twoeths committed Dec 9, 2021

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit f60786a5460cc323adeb2cfeec7df2f523349d17
13 changes: 4 additions & 9 deletions packages/cli/src/options/beaconNodeOptions/network.ts
Original file line number Diff line number Diff line change
@@ -12,7 +12,6 @@ export interface INetworkArgs {
"network.subscribeAllSubnets": boolean;
"network.connectToDiscv5Bootnodes": boolean;
"network.discv5FirstQueryDelayMs": number;
"network.requestCountTotalLimit": number;
"network.requestCountPeerLimit": number;
"network.blockCountTotalLimit": number;
"network.blockCountPeerLimit": number;
@@ -35,7 +34,6 @@ export function parseArgs(args: INetworkArgs): IBeaconNodeOptions["network"] {
subscribeAllSubnets: args["network.subscribeAllSubnets"],
connectToDiscv5Bootnodes: args["network.connectToDiscv5Bootnodes"],
discv5FirstQueryDelayMs: args["network.discv5FirstQueryDelayMs"],
requestCountTotalLimit: args["network.requestCountTotalLimit"],
requestCountPeerLimit: args["network.requestCountPeerLimit"],
blockCountTotalLimit: args["network.blockCountTotalLimit"],
blockCountPeerLimit: args["network.blockCountPeerLimit"],
@@ -115,37 +113,34 @@ export const options: ICliCommandOptions<INetworkArgs> = {
group: "network",
},

"network.requestCountTotalLimit": {
type: "number",
description: "Max block req/resp requests per rateTrackerTimeoutMs",
defaultDescription: String(defaultOptions.network.requestCountTotalLimit),
group: "network",
},

"network.requestCountPeerLimit": {
type: "number",
description: "Max block req/resp requests per peer per rateTrackerTimeoutMs",
hidden: true,
defaultDescription: String(defaultOptions.network.requestCountPeerLimit),
group: "network",
},

"network.blockCountTotalLimit": {
type: "number",
description: "Max block count requested per rateTrackerTimeoutMs",
hidden: true,
defaultDescription: String(defaultOptions.network.blockCountTotalLimit),
group: "network",
},

"network.blockCountPeerLimit": {
type: "number",
description: "Max block count requested per peer per rateTrackerTimeoutMs",
hidden: true,
defaultDescription: String(defaultOptions.network.blockCountPeerLimit),
group: "network",
},

"network.rateTrackerTimeoutMs": {
type: "number",
twoeths marked this conversation as resolved.
Show resolved Hide resolved
description: "Time window to track rate limit in milli seconds",
hidden: true,
defaultDescription: String(defaultOptions.network.rateTrackerTimeoutMs),
group: "network",
},
2 changes: 0 additions & 2 deletions packages/cli/test/unit/options/beaconNodeOptions.test.ts
Original file line number Diff line number Diff line change
@@ -46,7 +46,6 @@ describe("options / beaconNodeOptions", () => {
"network.subscribeAllSubnets": true,
"network.connectToDiscv5Bootnodes": true,
"network.discv5FirstQueryDelayMs": 1000,
"network.requestCountTotalLimit": 10,
"network.requestCountPeerLimit": 5,
"network.blockCountTotalLimit": 1000,
"network.blockCountPeerLimit": 500,
@@ -106,7 +105,6 @@ describe("options / beaconNodeOptions", () => {
subscribeAllSubnets: true,
connectToDiscv5Bootnodes: true,
discv5FirstQueryDelayMs: 1000,
requestCountTotalLimit: 10,
requestCountPeerLimit: 5,
blockCountTotalLimit: 1000,
blockCountPeerLimit: 500,
3 changes: 0 additions & 3 deletions packages/lodestar/src/network/network.ts
Original file line number Diff line number Diff line change
@@ -25,7 +25,6 @@ import {PeerManager} from "./peers/peerManager";
import {IPeerRpcScoreStore, PeerRpcScoreStore} from "./peers";
import {INetworkEventBus, NetworkEventBus} from "./events";
import {AttnetsService, SyncnetsService, CommitteeSubscription} from "./subnets";
import {InboundRateLimiter} from "./reqresp/response/rateLimiter";

interface INetworkModules {
config: IBeaconConfig;
@@ -69,7 +68,6 @@ export class Network implements INetwork {
const metadata = new MetadataController({}, {config, chain, logger});
const peerMetadata = new Libp2pPeerMetadataStore(libp2p.peerStore.metadataBook);
const peerRpcScores = new PeerRpcScoreStore(peerMetadata);
const inboundRateLimiter = new InboundRateLimiter(opts, {...modules, peerRpcScores});
this.events = networkEventBus;
this.metadata = metadata;
this.peerRpcScores = peerRpcScores;
@@ -82,7 +80,6 @@ export class Network implements INetwork {
peerMetadata,
metadata,
peerRpcScores,
inboundRateLimiter,
logger,
networkEventBus,
metrics,
11 changes: 2 additions & 9 deletions packages/lodestar/src/network/options.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import {ENR, IDiscv5DiscoveryInputOptions} from "@chainsafe/discv5";
import {PeerManagerOpts} from "./peers";
import {RateLimiterOpts} from "./reqresp/response/rateLimiter";
import {defaultRateLimiterOpts, RateLimiterOpts} from "./reqresp/response/rateLimiter";

export interface INetworkOptions extends PeerManagerOpts, RateLimiterOpts {
localMultiaddrs: string[];
@@ -23,13 +23,6 @@ export const defaultNetworkOptions: INetworkOptions = {
discv5FirstQueryDelayMs: 1000,
localMultiaddrs: ["/ip4/0.0.0.0/tcp/9000"],
bootMultiaddrs: [],
// rate tracker options per 1 minute
// per peer per minute, allow to serve up to 5 requests and 500 blocks
// total: make 4x peer params
requestCountTotalLimit: 200,
requestCountPeerLimit: 50,
blockCountTotalLimit: 2000,
blockCountPeerLimit: 500,
rateTrackerTimeoutMs: 60 * 1000,
discv5: defaultDiscv5Options,
...defaultRateLimiterOpts,
};
2 changes: 1 addition & 1 deletion packages/lodestar/src/network/peers/peerManager.ts
Original file line number Diff line number Diff line change
@@ -492,7 +492,7 @@ export class PeerManager {

this.logger.verbose("peer disconnected", {peer: prettyPrintPeerId(peer), direction, status});
this.networkEventBus.emit(NetworkEvent.peerDisconnected, peer);
this.reqResp.prune(peer);
this.reqResp.pruneRateLimiterData(peer);
this.metrics?.peerDisconnectedEvent.inc({direction});
};

6 changes: 3 additions & 3 deletions packages/lodestar/src/network/reqresp/interface.ts
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ import {MetadataController} from "../metadata";
import {INetworkEventBus} from "../events";
import {ReqRespHandlers} from "./handlers";
import {IMetrics} from "../../metrics";
import {RequestTypedContainer} from "./types";

export interface IReqResp {
start(): void;
@@ -22,7 +23,7 @@ export interface IReqResp {
request: phase0.BeaconBlocksByRangeRequest
): Promise<allForks.SignedBeaconBlock[]>;
beaconBlocksByRoot(peerId: PeerId, request: phase0.BeaconBlocksByRootRequest): Promise<allForks.SignedBeaconBlock[]>;
prune(peerId: PeerId): void;
pruneRateLimiterData(peerId: PeerId): void;
}

export interface IReqRespModules {
@@ -33,7 +34,6 @@ export interface IReqRespModules {
reqRespHandlers: ReqRespHandlers;
peerMetadata: IPeerMetadataStore;
peerRpcScores: IPeerRpcScoreStore;
inboundRateLimiter: IRateLimiter;
networkEventBus: INetworkEventBus;
metrics: IMetrics | null;
}
@@ -79,7 +79,7 @@ export interface IRateLimiter {
/**
* Allow to request or response based on rate limit params configured.
*/
allowRequest(peerId: PeerId, objectCount?: number): boolean;
allowRequest(peerId: PeerId, requestTyped: RequestTypedContainer): boolean;

/**
* Prune by peer id
6 changes: 4 additions & 2 deletions packages/lodestar/src/network/reqresp/rateTracker.ts
Original file line number Diff line number Diff line change
@@ -5,6 +5,8 @@ type RateTrackerOpts = {
timeoutMs: number;
};

const BUCKET_SIZE_MS = 1000;

/**
* The generic rate tracker allows up to `limit` objects in a period of time.
* This could apply to both request count or block count, for both requests and responses.
@@ -30,7 +32,7 @@ export class RateTracker {
}

this.requestsWithinWindow += objectCount;
const key = Math.floor(Date.now() / 1000);
const key = Math.floor(Date.now() / BUCKET_SIZE_MS);
const curObjectCount = this.requests.getOrDefault(key);
this.requests.set(key, curObjectCount + objectCount);

@@ -46,7 +48,7 @@ export class RateTracker {

for (const [timeInSec, count] of this.requests.entries()) {
// reclaim the quota for old requests
if (now - timeInSec * 1000 >= this.timeoutMs) {
if (now - timeInSec * BUCKET_SIZE_MS >= this.timeoutMs) {
this.requestsWithinWindow -= count;
this.requests.delete(timeInSec);
} else {
31 changes: 8 additions & 23 deletions packages/lodestar/src/network/reqresp/reqResp.ts
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@ import {
protocolsSupported,
IncomingResponseBody,
} from "./types";
import {InboundRateLimiter, RateLimiterOpts} from "./response/rateLimiter";

export type IReqRespOptions = Partial<typeof timeoutOptions>;

@@ -57,15 +58,15 @@ export class ReqResp implements IReqResp {
private respCount = 0;
private metrics: IMetrics | null;

constructor(modules: IReqRespModules, options?: IReqRespOptions) {
constructor(modules: IReqRespModules, options: IReqRespOptions & RateLimiterOpts) {
this.config = modules.config;
this.libp2p = modules.libp2p;
this.logger = modules.logger;
this.reqRespHandlers = modules.reqRespHandlers;
this.peerMetadata = modules.peerMetadata;
this.metadataController = modules.metadata;
this.peerRpcScores = modules.peerRpcScores;
this.inboundRateLimiter = modules.inboundRateLimiter;
this.inboundRateLimiter = new InboundRateLimiter(options, {...modules});
this.networkEventBus = modules.networkEventBus;
this.options = options;
this.metrics = modules.metrics;
@@ -94,7 +95,6 @@ export class ReqResp implements IReqResp {

async goodbye(peerId: PeerId, request: phase0.Goodbye): Promise<void> {
await this.sendRequest<phase0.Goodbye>(peerId, Method.Goodbye, [Version.V1], request);
this.inboundRateLimiter.prune(peerId);
}

async ping(peerId: PeerId): Promise<phase0.Ping> {
@@ -135,7 +135,7 @@ export class ReqResp implements IReqResp {
);
}

prune(peerId: PeerId): void {
pruneRateLimiterData(peerId: PeerId): void {
this.inboundRateLimiter.prune(peerId);
}

@@ -220,47 +220,32 @@ export class ReqResp implements IReqResp {
): AsyncIterable<OutgoingResponseBody> {
const requestTyped = {method: protocol.method, body: requestBody} as RequestTypedContainer;

if (requestTyped.method !== Method.Goodbye && !this.inboundRateLimiter.allowRequest(peerId, requestTyped)) {
throw new ResponseError(RespStatus.RATE_LIMITED, "rate limit");
}

switch (requestTyped.method) {
case Method.Ping:
if (!this.inboundRateLimiter.allowRequest(peerId)) {
throw new ResponseError(RespStatus.RATE_LIMITED, "rate limit");
}
yield this.metadataController.seqNumber;
break;
case Method.Metadata:
if (!this.inboundRateLimiter.allowRequest(peerId)) {
throw new ResponseError(RespStatus.RATE_LIMITED, "rate limit");
}
// V1 -> phase0, V2 -> altair. But the type serialization of phase0.Metadata will just ignore the extra .syncnets property
// It's safe to return altair.Metadata here for all versions
yield this.metadataController.json;
break;
case Method.Goodbye:
// no need to call this.inboundRateLimiter.allowRequest
this.inboundRateLimiter.prune(peerId);
yield BigInt(0);
break;

// Don't bubble Ping, Metadata, and, Goodbye requests to the app layer

case Method.Status:
if (!this.inboundRateLimiter.allowRequest(peerId)) {
throw new ResponseError(RespStatus.RATE_LIMITED, "rate limit");
}
yield* this.reqRespHandlers.onStatus();
break;
case Method.BeaconBlocksByRange:
if (!this.inboundRateLimiter.allowRequest(peerId, Math.max(requestTyped.body.count, 0))) {
throw new ResponseError(RespStatus.RATE_LIMITED, "rate limit");
}

yield* this.reqRespHandlers.onBeaconBlocksByRange(requestTyped.body);
break;
case Method.BeaconBlocksByRoot:
if (!this.inboundRateLimiter.allowRequest(peerId, requestTyped.body.length)) {
throw new ResponseError(RespStatus.RATE_LIMITED, "rate limit");
}

yield* this.reqRespHandlers.onBeaconBlocksByRoot(requestTyped.body);
break;

53 changes: 40 additions & 13 deletions packages/lodestar/src/network/reqresp/response/rateLimiter.ts
Original file line number Diff line number Diff line change
@@ -4,36 +4,57 @@ import {MapDef} from "../../../util/map";
import {IPeerRpcScoreStore, PeerAction} from "../../peers/score";
import {IRateLimiter} from "../interface";
import {RateTracker} from "../rateTracker";
import {Method, RequestTypedContainer} from "../types";

interface IRateLimiterModules {
logger: ILogger;
peerRpcScores: IPeerRpcScoreStore;
}

/**
* Options:
* - requestCountPeerLimit: maximum request count we can serve per peer within rateTrackerTimeoutMs
* - blockCountPeerLimit: maximum block count we can serve per peer within rateTrackerTimeoutMs
* - blockCountTotalLimit: maximum block count we can serve for all peers within rateTrackerTimeoutMs
* - rateTrackerTimeoutMs: the time period we want to track total requests or objects, normally 1 min
*/
export type RateLimiterOpts = {
requestCountTotalLimit: number;
requestCountPeerLimit: number;
blockCountTotalLimit: number;
blockCountPeerLimit: number;
blockCountTotalLimit: number;
rateTrackerTimeoutMs: number;
};

/**
* The rate tracker for all peers.
* Default value for RateLimiterOpts
* - requestCountPeerLimit: allow to serve 50 requests per peer within 1 minute
* - blockCountPeerLimit: allow to serve 500 blocks per peer within 1 minute
* - blockCountTotalLimit: allow to serve 2000 (blocks) for all peer within 1 minute (4 x blockCountPeerLimit)
* - rateTrackerTimeoutMs: 1 minute
*/
export const defaultRateLimiterOpts = {
requestCountPeerLimit: 50,
blockCountPeerLimit: 500,
blockCountTotalLimit: 2000,
rateTrackerTimeoutMs: 60 * 1000,
};

/**
* This class is singleton, it has per-peer request count rate tracker and block count rate tracker
* and a block count rate tracker for all peers (this is lodestar specific).
*/
export class InboundRateLimiter implements IRateLimiter {
private readonly logger: ILogger;
private readonly peerRpcScores: IPeerRpcScoreStore;
private requestCountTotalTracker: RateTracker;
private requestCountTrackersByPeer: MapDef<string, RateTracker>;
/**
* This rate tracker is specific to lodestar, we don't want to serve too many blocks for peers at the
* same time, even through we limit block count per peer as in blockCountTrackersByPeer
*/
private blockCountTotalTracker: RateTracker;
twoeths marked this conversation as resolved.
Show resolved Hide resolved
private blockCountTrackersByPeer: MapDef<string, RateTracker>;

constructor(opts: RateLimiterOpts, modules: IRateLimiterModules) {
this.requestCountTotalTracker = new RateTracker({
limit: opts.requestCountTotalLimit,
timeoutMs: opts.rateTrackerTimeoutMs,
});
this.requestCountTrackersByPeer = new MapDef(
() => new RateTracker({limit: opts.requestCountPeerLimit, timeoutMs: opts.rateTrackerTimeoutMs})
);
@@ -50,9 +71,8 @@ export class InboundRateLimiter implements IRateLimiter {

/**
* Tracks a request from a peer and returns whether to allow the request based on the configured rate limit params.
* @param numBlock only applies to beacon_blocks_by_range and beacon_blocks_by_root
*/
allowRequest(peerId: PeerId, numBlock?: number): boolean {
allowRequest(peerId: PeerId, requestTyped: RequestTypedContainer): boolean {
const peerIdStr = peerId.toB58String();

// rate limit check for request
@@ -66,12 +86,18 @@ export class InboundRateLimiter implements IRateLimiter {
return false;
}

if (this.requestCountTotalTracker.requestObjects(1) === 0) {
return false;
let numBlock = 0;
switch (requestTyped.method) {
case Method.BeaconBlocksByRange:
numBlock = requestTyped.body.count;
break;
case Method.BeaconBlocksByRoot:
numBlock = requestTyped.body.length;
break;
}

// rate limit check for block count
if (numBlock !== undefined) {
if (numBlock > 0) {
const blockCountPeerTracker = this.blockCountTrackersByPeer.getOrDefault(peerIdStr);
if (blockCountPeerTracker.requestObjects(numBlock) === 0) {
this.logger.verbose("Do not serve block request due to block count rate limit", {
@@ -84,6 +110,7 @@ export class InboundRateLimiter implements IRateLimiter {
}

if (this.blockCountTotalTracker.requestObjects(numBlock) === 0) {
// don't apply penalty
return false;
}
}
Loading