Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Public API IP rate limiting #4764

Merged
merged 7 commits into from
Mar 3, 2022
Merged

Public API IP rate limiting #4764

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b1167fd
Adding public API rate limiting, with env variable option, defaults t…
mike12345567 Mar 2, 2022
a8a8498
Updating rate limiter to make use of Redis.
mike12345567 Mar 2, 2022
32e7f27
Adding check to disable rate limit redis connection in test.
mike12345567 Mar 2, 2022
f01a072
Merge branch 'develop' of github.com:Budibase/budibase into feature/r…
mike12345567 Mar 3, 2022
216af35
Fixing review comments.
mike12345567 Mar 3, 2022
54137ec
Removing koa rate limit types, was requiring types for every store, t…
mike12345567 Mar 3, 2022
86f649c
Upping test rate limit level.
mike12345567 Mar 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/server/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@
"koa-send": "5.0.0",
"koa-session": "5.12.0",
"koa-static": "5.0.0",
"koa2-ratelimit": "^1.1.0",
"lodash": "4.17.21",
"memorystream": "^0.3.1",
"mongodb": "3.6.3",
Expand All @@ -126,6 +127,7 @@
"pouchdb-all-dbs": "1.0.2",
"pouchdb-find": "^7.2.2",
"pouchdb-replication-stream": "1.2.9",
"redis": "4",
mike12345567 marked this conversation as resolved.
Show resolved Hide resolved
"server-destroy": "1.0.1",
"svelte": "^3.38.2",
"swagger-parser": "^10.0.3",
Expand Down
28 changes: 28 additions & 0 deletions packages/server/src/api/routes/public/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,46 @@ import authorized from "../../../middleware/authorized"
import { paramResource, paramSubResource } from "../../../middleware/resourceId"
import { CtxFn } from "./utils/Endpoint"
import mapperMiddleware from "./middleware/mapper"
import env from "../../../environment"
// below imports don't have declaration files
const Router = require("@koa/router")
const RateLimit = require("koa2-ratelimit").RateLimit
const Stores = require("koa2-ratelimit").Stores
mike12345567 marked this conversation as resolved.
Show resolved Hide resolved
const {
PermissionLevels,
PermissionTypes,
} = require("@budibase/backend-core/permissions")
const { getRedisOptions } = require("@budibase/backend-core/redis").utils

const PREFIX = "/api/public/v1"
const DEFAULT_API_LIMITING = 120
mike12345567 marked this conversation as resolved.
Show resolved Hide resolved

if (!env.isTest()) {
const REDIS_OPTS = getRedisOptions()
RateLimit.defaultOptions({
store: new Stores.Redis({
socket: {
host: REDIS_OPTS.host,
port: REDIS_OPTS.port,
},
password: REDIS_OPTS.opts.password,
database: 1,
}),
})
}
// rate limiting, allows for 2 requests per second
const limiter = RateLimit.middleware({
interval: { min: 1 },
// per ip, per interval
max: env.API_RATE_LIMITING || DEFAULT_API_LIMITING,
})

const publicRouter = new Router({
prefix: PREFIX,
})

publicRouter.use(limiter)

function addMiddleware(
endpoints: any,
middleware: CtxFn,
Expand Down
1 change: 1 addition & 0 deletions packages/server/src/environment.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ module.exports = {
INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
MULTI_TENANCY: process.env.MULTI_TENANCY,
HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
API_RATE_LIMITING: process.env.API_RATE_LIMITING,
mike12345567 marked this conversation as resolved.
Show resolved Hide resolved
// environment
NODE_ENV: process.env.NODE_ENV,
JEST_WORKER_ID: process.env.JEST_WORKER_ID,
Expand Down
113 changes: 85 additions & 28 deletions packages/server/yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -995,10 +995,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==

"@budibase/backend-core@^1.0.79-alpha.5":
version "1.0.79-alpha.5"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.79-alpha.5.tgz#dac73ccfcd2e6e63415cde6d76e4bf09043dc6b6"
integrity sha512-m7/z55fp+EYVYEAyuQ2K//AcrfgzLBcR4EVjP+rrmbIhGFbLV2ASl5IBg9bcAYp0z2m816skJrY2asx9raWhgw==
"@budibase/backend-core@^1.0.79-alpha.7":
version "1.0.79-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.79-alpha.7.tgz#57e8319118b425cc228173d1ec8bf19843e1a417"
integrity sha512-Ao4dR6zwnJa4mYiRyl5lULYm+wsYwOi3sxDIjt5vmurqUL2JTuVUrGt1MAq1N6K11xQpbQQJAFvgfxBgn9aEMg==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
Expand Down Expand Up @@ -1068,7 +1068,7 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"

"@budibase/bbui@^1.0.79-alpha.5":
"@budibase/bbui@^1.0.79-alpha.7":
version "1.58.13"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.58.13.tgz#59df9c73def2d81c75dcbd2266c52c19db88dbd7"
integrity sha512-Zk6CKXdBfKsTVzA1Xs5++shdSSZLfphVpZuKVbjfzkgtuhyH7ruucexuSHEpFsxjW5rEKgKIBoRFzCK5vPvN0w==
Expand All @@ -1080,14 +1080,14 @@
svelte-portal "^1.0.0"
turndown "^7.0.0"

"@budibase/client@^1.0.79-alpha.5":
version "1.0.79-alpha.5"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.79-alpha.5.tgz#d729858b10e6cd2a506fb63364a0e7ab3149780e"
integrity sha512-OrBErU97YL67GggsLmcD46AUElSgtyFjZdCXi++3s4zaZYZxT4Ix2iFMrnslcpF87bv8xyiSt3vsyCPGKCU5wQ==
"@budibase/client@^1.0.79-alpha.7":
version "1.0.79-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.79-alpha.7.tgz#d225ac5bd68fa9ecb81114791e6d931246da9637"
integrity sha512-7faCcIlXyOf660PwpOMCt9/X2liiTuCsPGUpLsJQu2j9CcVZ5vV+au0CX7dtqewtPNuIL0mF3G7ZOpBTvXx4NQ==
dependencies:
"@budibase/bbui" "^1.0.79-alpha.5"
"@budibase/frontend-core" "^1.0.79-alpha.5"
"@budibase/string-templates" "^1.0.79-alpha.5"
"@budibase/bbui" "^1.0.79-alpha.7"
"@budibase/frontend-core" "^1.0.79-alpha.7"
"@budibase/string-templates" "^1.0.79-alpha.7"
"@spectrum-css/button" "^3.0.3"
"@spectrum-css/card" "^3.0.3"
"@spectrum-css/divider" "^1.0.3"
Expand All @@ -1106,12 +1106,12 @@
svelte-flatpickr "^3.1.0"
svelte-spa-router "^3.0.5"

"@budibase/frontend-core@^1.0.79-alpha.5":
version "1.0.79-alpha.5"
resolved "https://registry.yarnpkg.com/@budibase/frontend-core/-/frontend-core-1.0.79-alpha.5.tgz#7da5faf83d6cc5a59d8e038c2e9333e27bff35d5"
integrity sha512-5xti0MdKRvNKwYUE5cp4rH8IwLPmuRz39ajck947ut2OWzXV9bt7SXzoKPSSzEGdCBA2DgzJpK3gQWYlqXiJiQ==
"@budibase/frontend-core@^1.0.79-alpha.7":
version "1.0.79-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/frontend-core/-/frontend-core-1.0.79-alpha.7.tgz#cba8f61932f966dc3f19cc7d5fed45d832ee676e"
integrity sha512-mEspQXLUnjvNcL7QfDN1qIFGRo+AfdcaEq23gKAWXF1R+Byy7VCYDzcowzJY/TT6B4BSq3z6s57z3ILKtqI7zA==
dependencies:
"@budibase/bbui" "^1.0.79-alpha.5"
"@budibase/bbui" "^1.0.79-alpha.7"
lodash "^4.17.21"
svelte "^3.46.2"

Expand Down Expand Up @@ -1158,10 +1158,10 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"

"@budibase/string-templates@^1.0.79-alpha.5":
version "1.0.79-alpha.5"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.79-alpha.5.tgz#063f5beca7d3b4a9757df77dcf1bd8a442d7522e"
integrity sha512-Rifn1h1Pn53KYCFX6GHmMq+fD4IEnfRXEWrf4RD7cy4TVCYqCIcI84tnzUwibkyuCbpDw4zh0RR0m4nemf7heg==
"@budibase/string-templates@^1.0.79-alpha.7":
version "1.0.79-alpha.7"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.79-alpha.7.tgz#3e5235e05f13fe406cae62862110f841788d1bc0"
integrity sha512-wdnk0wi9vuSYY7vimIGV1+i0dSONOBg5deZia8v9O8XM9OmJohLUIkJdMNhhv9OCxyeC53gauaxhVdKeop6kmA==
dependencies:
"@budibase/handlebars-helpers" "^0.11.8"
dayjs "^1.10.4"
Expand Down Expand Up @@ -1885,6 +1885,41 @@
path-to-regexp "^1.1.1"
urijs "^1.19.0"

"@node-redis/bloom@1.0.1":
version "1.0.1"
resolved "https://registry.yarnpkg.com/@node-redis/bloom/-/bloom-1.0.1.tgz#144474a0b7dc4a4b91badea2cfa9538ce0a1854e"
integrity sha512-mXEBvEIgF4tUzdIN89LiYsbi6//EdpFA7L8M+DHCvePXg+bfHWi+ct5VI6nHUFQE5+ohm/9wmgihCH3HSkeKsw==

"@node-redis/client@1.0.4":
version "1.0.4"
resolved "https://registry.yarnpkg.com/@node-redis/client/-/client-1.0.4.tgz#fe185750df3bcc07524f63fe8dbc8d14d22d6cbb"
integrity sha512-IM/NRAqg7MvNC3bIRQipXGrEarunrdgvrbAzsd3ty93LSHi/M+ybQulOERQi8a3M+P5BL8HenwXjiIoKm6ml2g==
dependencies:
cluster-key-slot "1.1.0"
generic-pool "3.8.2"
redis-parser "3.0.0"
yallist "4.0.0"

"@node-redis/graph@1.0.0":
version "1.0.0"
resolved "https://registry.yarnpkg.com/@node-redis/graph/-/graph-1.0.0.tgz#baf8eaac4a400f86ea04d65ec3d65715fd7951ab"
integrity sha512-mRSo8jEGC0cf+Rm7q8mWMKKKqkn6EAnA9IA2S3JvUv/gaWW/73vil7GLNwion2ihTptAm05I9LkepzfIXUKX5g==

"@node-redis/json@1.0.2":
version "1.0.2"
resolved "https://registry.yarnpkg.com/@node-redis/json/-/json-1.0.2.tgz#8ad2d0f026698dc1a4238cc3d1eb099a3bee5ab8"
integrity sha512-qVRgn8WfG46QQ08CghSbY4VhHFgaTY71WjpwRBGEuqGPfWwfRcIf3OqSpR7Q/45X+v3xd8mvYjywqh0wqJ8T+g==

"@node-redis/search@1.0.3":
version "1.0.3"
resolved "https://registry.yarnpkg.com/@node-redis/search/-/search-1.0.3.tgz#7c3d026bf994caf82019fd0c3924cfc09f041a29"
integrity sha512-rsrzkGWI84di/uYtEctS/4qLusWt0DESx/psjfB0TFpORDhe7JfC0h8ary+eHulTksumor244bXLRSqQXbFJmw==

"@node-redis/time-series@1.0.2":
version "1.0.2"
resolved "https://registry.yarnpkg.com/@node-redis/time-series/-/time-series-1.0.2.tgz#5dd3638374edd85ebe0aa6b0e87addc88fb9df69"
integrity sha512-HGQ8YooJ8Mx7l28tD7XjtB3ImLEjlUxG1wC1PAjxu6hPJqjPshUZxAICzDqDjtIbhDTf48WXXUcx8TQJB1XTKA==

"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
Expand Down Expand Up @@ -4124,7 +4159,7 @@ cls-hooked@^4.2.2:
emitter-listener "^1.0.1"
semver "^5.4.1"

cluster-key-slot@^1.1.0:
cluster-key-slot@1.1.0, cluster-key-slot@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==
Expand Down Expand Up @@ -6103,6 +6138,11 @@ generate-function@^2.3.1:
dependencies:
is-property "^1.0.2"

generic-pool@3.8.2:
version "3.8.2"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.8.2.tgz#aab4f280adb522fdfbdc5e5b64d718d3683f04e9"
integrity sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==

gensync@^1.0.0-beta.2:
version "1.0.0-beta.2"
resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
Expand Down Expand Up @@ -8707,6 +8747,11 @@ koa-views@^7.0.1:
pretty "^2.0.0"
resolve-path "^1.4.0"

koa2-ratelimit@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/koa2-ratelimit/-/koa2-ratelimit-1.1.0.tgz#5ab432fdda7b2d63a4fb1b9a0d994c1264396aff"
integrity sha512-AumRCI8YO9TMF9trVP6j68K5qzi21ajZUOCb5VuPWq9pZw+FHXam275S5P1IDAlZjs1cDFBOAAkhwTdTbVCcsg==

koa@2.7.0:
version "2.7.0"
resolved "https://registry.yarnpkg.com/koa/-/koa-2.7.0.tgz#7e00843506942b9d82c6cc33749f657c6e5e7adf"
Expand Down Expand Up @@ -11176,13 +11221,25 @@ redis-info@^3.0.8:
dependencies:
lodash "^4.17.11"

redis-parser@^3.0.0:
redis-parser@3.0.0, redis-parser@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4"
integrity sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=
dependencies:
redis-errors "^1.0.0"

redis@4:
version "4.0.4"
resolved "https://registry.yarnpkg.com/redis/-/redis-4.0.4.tgz#b567f82f59086df38433982f7f424b48e924ec7a"
integrity sha512-KaM1OAj/nGrSeybmmOWSMY0LXTGT6FVWgUZZrd2MYzXKJ+VGtqVaciGQeNMfZiQX+kDM8Ke4uttb54m2rm6V0A==
dependencies:
"@node-redis/bloom" "1.0.1"
"@node-redis/client" "1.0.4"
"@node-redis/graph" "1.0.0"
"@node-redis/json" "1.0.2"
"@node-redis/search" "1.0.3"
"@node-redis/time-series" "1.0.2"

regenerate-unicode-properties@^9.0.0:
version "9.0.0"
resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-9.0.0.tgz#54d09c7115e1f53dc2314a974b32c1c344efe326"
Expand Down Expand Up @@ -13712,16 +13769,16 @@ y18n@^5.0.5:
resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"
integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==

yallist@4.0.0, yallist@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==

yallist@^2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=

yallist@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==

yaml@2.0.0-1:
version "2.0.0-1"
resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.0.0-1.tgz#8c3029b3ee2028306d5bcf396980623115ff8d18"
Expand Down