diff --git a/.yarn/cache/@oclif-core-npm-3.26.5-02719845fd-4e2aa1a945.zip b/.yarn/cache/@oclif-core-npm-3.26.5-02719845fd-4e2aa1a945.zip new file mode 100644 index 0000000000..abfc68a753 Binary files /dev/null and b/.yarn/cache/@oclif-core-npm-3.26.5-02719845fd-4e2aa1a945.zip differ diff --git a/.yarn/cache/color-npm-4.2.3-4a23227581-b23f5e500a.zip b/.yarn/cache/color-npm-4.2.3-4a23227581-b23f5e500a.zip new file mode 100644 index 0000000000..3f65ebe0c1 Binary files /dev/null and b/.yarn/cache/color-npm-4.2.3-4a23227581-b23f5e500a.zip differ diff --git a/.yarn/cache/color-string-npm-1.9.1-dc020e56be-72aa0b81ee.zip b/.yarn/cache/color-string-npm-1.9.1-dc020e56be-72aa0b81ee.zip new file mode 100644 index 0000000000..7cb3fbdc1b Binary files /dev/null and b/.yarn/cache/color-string-npm-1.9.1-dc020e56be-72aa0b81ee.zip differ diff --git a/.yarn/cache/ejs-npm-3.1.10-4e8cf4bdc1-a9cb7d7cd1.zip b/.yarn/cache/ejs-npm-3.1.10-4e8cf4bdc1-a9cb7d7cd1.zip new file mode 100644 index 0000000000..da09b74bcf Binary files /dev/null and b/.yarn/cache/ejs-npm-3.1.10-4e8cf4bdc1-a9cb7d7cd1.zip differ diff --git a/.yarn/cache/minimatch-npm-9.0.4-7be5a33efc-4cdc18d112.zip b/.yarn/cache/minimatch-npm-9.0.4-7be5a33efc-4cdc18d112.zip new file mode 100644 index 0000000000..61a88c7c69 Binary files /dev/null and b/.yarn/cache/minimatch-npm-9.0.4-7be5a33efc-4cdc18d112.zip differ diff --git a/.yarn/cache/password-prompt-npm-1.1.3-0190666768-1cf7001e66.zip b/.yarn/cache/password-prompt-npm-1.1.3-0190666768-1cf7001e66.zip new file mode 100644 index 0000000000..11d8442b2b Binary files /dev/null and b/.yarn/cache/password-prompt-npm-1.1.3-0190666768-1cf7001e66.zip differ diff --git a/Cargo.lock b/Cargo.lock index a871a5be5d..96ce949078 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -234,9 +234,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit", "memchr", @@ -260,8 +260,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -1102,7 +1102,7 @@ dependencies = [ "envy", "futures", "hex", - "http", + "http 0.2.12", "lru", "rs-dapi-client", "sanitize-filename", @@ -1698,6 +1698,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1972,7 +1987,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", "indexmap 2.2.6", "slab", "tokio", @@ -1995,15 +2029,6 @@ dependencies = [ "ahash 0.7.8", ] -[[package]] -name = "hashbrown" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" -dependencies = [ - "ahash 0.8.11", -] - [[package]] name = "hashbrown" version = "0.14.3" @@ -2089,6 +2114,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -2096,7 +2132,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -2128,9 +2187,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -2142,18 +2201,75 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-timeout" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.3.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2537,56 +2653,47 @@ checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "metrics" -version = "0.21.1" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +checksum = "2be3cbd384d4e955b231c895ce10685e3d8260c5ccffae898c96c723b0772835" dependencies = [ "ahash 0.8.11", - "metrics-macros", "portable-atomic", ] [[package]] name = "metrics-exporter-prometheus" -version = "0.12.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950" +checksum = "5d58e362dc7206e9456ddbcdbd53c71ba441020e62104703075a69151e38d85f" dependencies = [ - "base64 0.21.7", - "hyper", - "indexmap 1.9.3", + "base64 0.22.0", + "http-body-util", + "hyper 1.3.1", + "hyper-tls", + "hyper-util", + "indexmap 2.2.6", "ipnet", "metrics", "metrics-util", - "quanta", + "quanta 0.12.3", "thiserror", "tokio", "tracing", ] -[[package]] -name = "metrics-macros" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] - [[package]] name = "metrics-util" -version = "0.15.1" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" +checksum = "8b07a5eb561b8cbc16be2d216faf7757f9baf3bfb94dbb0fae3df8387a5bb47f" dependencies = [ "crossbeam-epoch", "crossbeam-utils", - "hashbrown 0.13.1", + "hashbrown 0.14.3", "metrics", "num_cpus", - "quanta", + "quanta 0.12.3", "sketches-ddsketch", ] @@ -2663,7 +2770,7 @@ dependencies = [ "futures-util", "once_cell", "parking_lot", - "quanta", + "quanta 0.11.1", "rustc_version", "scheduled-thread-pool", "skeptic", @@ -2692,6 +2799,24 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -2874,12 +2999,50 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "overload" version = "0.1.1" @@ -3382,7 +3545,22 @@ dependencies = [ "libc", "mach2", "once_cell", - "raw-cpuid", + "raw-cpuid 10.7.0", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quanta" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid 11.0.1", "wasi", "web-sys", "winapi", @@ -3442,6 +3620,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "raw-cpuid" +version = "11.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "rayon" version = "1.10.0" @@ -3596,7 +3783,7 @@ dependencies = [ "dapi-grpc", "futures", "hex", - "http", + "http 0.2.12", "lru", "rand", "sha2", @@ -4529,6 +4716,16 @@ dependencies = [ "syn 2.0.60", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.25.0" @@ -4645,10 +4842,10 @@ dependencies = [ "axum", "base64 0.21.7", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -4672,10 +4869,10 @@ dependencies = [ "axum", "base64 0.21.7", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -4754,6 +4951,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", diff --git a/Dockerfile b/Dockerfile index 7cafb4713d..a7c3030302 100644 --- a/Dockerfile +++ b/Dockerfile @@ -194,7 +194,8 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \ cargo build \ --profile "$CARGO_BUILD_PROFILE" \ - --package drive-abci && \ + --package drive-abci \ + --locked && \ cp /platform/target/*/drive-abci /artifacts/ && \ if [[ "${RUSTC_WRAPPER}" == "sccache" ]] ; then sccache --show-stats; fi diff --git a/packages/dashmate/README.md b/packages/dashmate/README.md index acfa37c0c5..74b21560e6 100644 --- a/packages/dashmate/README.md +++ b/packages/dashmate/README.md @@ -71,11 +71,11 @@ $ dashmate update ║ Drive ABCI │ dashpay/drive:0.24 │ updated ║ ║ Drive Tenderdash │ dashpay/tenderdash:0.11.2 │ up to date ║ ║ DAPI API │ dashpay/dapi:0.24 │ updated ║ -║ DAPI Envoy │ dashpay/envoy:0.24 │ updated ║ +║ Gateway │ dashpay/envoy:0.24 │ updated ║ ║ Dashmate Helper │ dashpay/dashmate-helper:0.24 │ updated ║ ╚══════════════════╧══════════════════════════════╧════════════╝ $ dashmate update --format=json -[{"name":"core","title":"Core","updated":false,"image":"dashpay/dashd:19.2.0"},{"name":"drive_abci","title":"Drive ABCI","pulled":false,"image":"dashpay/drive:0.24"},{"name":"drive_tenderdash","title":"Drive Tenderdash","pulled":true,"image":"dashpay/tenderdash:0.11.2"},{"name":"dapi_api","title":"DAPI API","pulled":false,"image":"dashpay/dapi:0.24"},{"name":"dapi_envoy","title":"DAPI Envoy","pulled":false,"image":"dashpay/envoy:0.24"},{"name":"dashmate_helper","title":"Dashmate Helper","pulled":false,"image":"dashpay/dashmate-helper:0.24"}] +[{"name":"core","title":"Core","updated":false,"image":"dashpay/dashd:19.2.0"},{"name":"drive_abci","title":"Drive ABCI","pulled":false,"image":"dashpay/drive:0.24"},{"name":"drive_tenderdash","title":"Drive Tenderdash","pulled":true,"image":"dashpay/tenderdash:0.11.2"},{"name":"dapi_api","title":"DAPI API","pulled":false,"image":"dashpay/dapi:0.24"},{"name":"gateway","title":"Gateway","pulled":false,"image":"dashpay/envoy:0.24"},{"name":"dashmate_helper","title":"Dashmate Helper","pulled":false,"image":"dashpay/dashmate-helper:0.24"}] $ dashmate start ``` @@ -503,7 +503,7 @@ again all your service configs (dashd.conf, config.toml, etc.), you can issue th ```bash dashmate config render -Config "testnet" service configs rendered +"testnet" service configs rendered ``` ### Development diff --git a/packages/dashmate/configs/defaults/getBaseConfigFactory.js b/packages/dashmate/configs/defaults/getBaseConfigFactory.js index dfa5800145..9459d657db 100644 --- a/packages/dashmate/configs/defaults/getBaseConfigFactory.js +++ b/packages/dashmate/configs/defaults/getBaseConfigFactory.js @@ -125,34 +125,85 @@ export default function getBaseConfigFactory(homeDir) { indexes: true, }, platform: { - dapi: { - envoy: { - docker: { - image: 'dashpay/envoy:1.22.11', + gateway: { + docker: { + image: 'dashpay/envoy:1.30.2-impr.1', + }, + maxConnections: 1000, + maxHeapSizeInBytes: 125000000, // 1 Gb + upstreams: { + driveGrpc: { + maxRequests: 100, }, - http: { + dapiApi: { + maxRequests: 100, + }, + dapiCoreStreams: { + maxRequests: 100, + }, + dapiJsonRpc: { + maxRequests: 100, + }, + }, + metrics: { + enabled: false, + host: '127.0.0.1', + port: 9090, + }, + admin: { + enabled: false, + host: '127.0.0.1', + port: 9901, + }, + listeners: { + dapiAndDrive: { + http2: { + maxConcurrentStreams: 10, + }, host: '0.0.0.0', port: 443, - connectTimeout: '5s', - responseTimeout: '15s', }, - rateLimiter: { - maxTokens: 300, - tokensPerFill: 150, - fillInterval: '60s', - enabled: true, + }, + log: { + level: 'info', + accessLogs: [ + { + type: 'stdout', + format: 'text', + template: null, + }, + ], + }, + rateLimiter: { + docker: { + image: 'envoyproxy/ratelimit:3fcc3609', }, - ssl: { + metrics: { enabled: false, - provider: 'zerossl', - providerConfigs: { - zerossl: { - apiKey: null, - id: null, - }, + docker: { + image: 'prom/statsd-exporter:v0.26.1', }, + host: '127.0.0.1', + port: 9102, }, + unit: 'minute', + requestsPerUnit: 150, + blacklist: [], + whitelist: [], + enabled: true, }, + ssl: { + enabled: false, + provider: 'zerossl', + providerConfigs: { + zerossl: { + apiKey: null, + id: null, + }, + }, + }, + }, + dapi: { api: { docker: { image: `dashpay/dapi:${dockerImageVersion}`, @@ -191,7 +242,7 @@ export default function getBaseConfigFactory(homeDir) { enabled: false, host: '127.0.0.1', port: 6669, - retention_secs: 60 * 3, + retention: 60 * 3, }, validatorSet: { llmqType: 4, @@ -201,6 +252,11 @@ export default function getBaseConfigFactory(homeDir) { dkgInterval: 288, llmqSize: 400, }, + metrics: { + enabled: false, + host: '127.0.0.1', + port: 29090, + }, epochTime: 788400, }, tenderdash: { diff --git a/packages/dashmate/configs/defaults/getLocalConfigFactory.js b/packages/dashmate/configs/defaults/getLocalConfigFactory.js index 88b2870fc9..33992ceceb 100644 --- a/packages/dashmate/configs/defaults/getLocalConfigFactory.js +++ b/packages/dashmate/configs/defaults/getLocalConfigFactory.js @@ -36,17 +36,17 @@ export default function getLocalConfigFactory(getBaseConfig) { }, }, platform: { - dapi: { - envoy: { - ssl: { - provider: SSL_PROVIDERS.SELF_SIGNED, - }, - http: { + gateway: { + ssl: { + provider: SSL_PROVIDERS.SELF_SIGNED, + }, + listeners: { + dapiAndDrive: { port: 2443, }, - rateLimiter: { - enabled: false, - }, + }, + rateLimiter: { + enabled: false, }, }, drive: { diff --git a/packages/dashmate/configs/defaults/getTestnetConfigFactory.js b/packages/dashmate/configs/defaults/getTestnetConfigFactory.js index c885080be7..323d63314b 100644 --- a/packages/dashmate/configs/defaults/getTestnetConfigFactory.js +++ b/packages/dashmate/configs/defaults/getTestnetConfigFactory.js @@ -44,9 +44,9 @@ export default function getTestnetConfigFactory(homeDir, getBaseConfig) { }, }, platform: { - dapi: { - envoy: { - http: { + gateway: { + listeners: { + dapiAndDrive: { port: 1443, }, }, diff --git a/packages/dashmate/configs/getConfigFileMigrationsFactory.js b/packages/dashmate/configs/getConfigFileMigrationsFactory.js index 0dc4d7027f..cdd3cd9405 100644 --- a/packages/dashmate/configs/getConfigFileMigrationsFactory.js +++ b/packages/dashmate/configs/getConfigFileMigrationsFactory.js @@ -57,7 +57,7 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs) options.platform.dapi.api.docker.image = base.get('platform.dapi.api.docker.image'); - options.platform.dapi.envoy.docker.image = base.get('platform.dapi.envoy.docker.image'); + options.platform.gateway.docker.image = base.get('platform.gateway.docker.image'); }); return configFile; @@ -106,7 +106,7 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs) '0.24.16': (configFile) => { Object.entries(configFile.configs) .forEach(([, options]) => { - options.platform.dapi.envoy.docker = base.get('platform.dapi.envoy.docker'); + options.platform.gateway.docker = base.get('platform.gateway.docker'); options.platform.dapi.api.docker.build = base.get('platform.dapi.api.docker.build'); @@ -382,7 +382,7 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs) options.core.p2p.host = base.get('core.p2p.host'); options.core.rpc.host = base.get('core.rpc.host'); - options.platform.dapi.envoy.http.host = base.get('platform.dapi.envoy.http.host'); + options.platform.dapi.envoy.http.host = '0.0.0.0'; options.platform.drive.tenderdash.p2p.host = base.get('platform.drive.tenderdash.p2p.host'); options.platform.drive.tenderdash.rpc.host = base.get('platform.drive.tenderdash.rpc.host'); options.platform.drive.tenderdash.metrics.host = base.get('platform.drive.tenderdash.metrics.host'); @@ -401,8 +401,8 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs) '0.25.20': (configFile) => { Object.entries(configFile.configs) .forEach(([name, options]) => { - options.platform.dapi.envoy.http.connectTimeout = base.get('platform.dapi.envoy.http.connectTimeout'); - options.platform.dapi.envoy.http.responseTimeout = base.get('platform.dapi.envoy.http.responseTimeout'); + options.platform.dapi.envoy.http.connectTimeout = '5s'; + options.platform.dapi.envoy.http.responseTimeout = '15s'; options.platform.drive.tenderdash.rpc.maxOpenConnections = base.get('platform.drive.tenderdash.rpc.maxOpenConnections'); @@ -523,9 +523,92 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs) }, '1.0.0-dev.12': (configFile) => { Object.entries(configFile.configs) - .forEach(([, options]) => { + .forEach(([name, options]) => { + // Update tenderdash config options.platform.drive.tenderdash.docker.image = base.get('platform.drive.tenderdash.docker.image'); options.platform.drive.tenderdash.mempool.maxConcurrentCheckTx = base.get('platform.drive.tenderdash.mempool.maxConcurrentCheckTx'); + + // Add metrics to Drive ABCI + options.platform.drive.abci.metrics = base.get('platform.drive.abci.metrics'); + + // Envoy -> Gateway + if (options.platform.dapi.envoy) { + options.platform.gateway = lodash.cloneDeep(options.platform.dapi.envoy); + + // add new options + options.platform.gateway.maxConnections = base.get('platform.gateway.maxConnections'); + options.platform.gateway.maxHeapSizeInBytes = base.get('platform.gateway.maxHeapSizeInBytes'); + options.platform.gateway.metrics = base.get('platform.gateway.metrics'); + options.platform.gateway.admin = base.get('platform.gateway.admin'); + options.platform.gateway.upstreams = base.get('platform.gateway.upstreams'); + options.platform.gateway.log = base.get('platform.gateway.log'); + + // http -> listeners + options.platform.gateway.listeners = lodash.cloneDeep( + base.get('platform.gateway.listeners'), + ); + + options.platform.gateway.listeners.dapiAndDrive.host = options.platform.dapi.envoy + .http.host; + options.platform.gateway.listeners.dapiAndDrive.port = options.platform.dapi.envoy + .http.port; + + delete options.platform.gateway.http; + + // update rate limiter + options.platform.gateway.rateLimiter.docker = base.get('platform.gateway.rateLimiter.docker'); + options.platform.gateway.rateLimiter.unit = base.get('platform.gateway.rateLimiter.unit'); + options.platform.gateway.rateLimiter.requestsPerUnit = base.get('platform.gateway.rateLimiter.requestsPerUnit'); + options.platform.gateway.rateLimiter.blacklist = base.get('platform.gateway.rateLimiter.blacklist'); + options.platform.gateway.rateLimiter.whitelist = base.get('platform.gateway.rateLimiter.whitelist'); + options.platform.gateway.rateLimiter.metrics = base.get('platform.gateway.rateLimiter.metrics'); + + delete options.platform.gateway.rateLimiter.fillInterval; + delete options.platform.gateway.rateLimiter.maxTokens; + delete options.platform.gateway.rateLimiter.tokensPerFill; + + // delete envoy + delete options.platform.dapi.envoy; + + // update image + options.platform.gateway.docker.image = base.get('platform.gateway.docker.image'); + } + + // rename non conventional field + if (options.platform.drive.abci.tokioConsole.retention_secs) { + options.platform.drive.abci.tokioConsole.retention = options.platform.drive.abci + .tokioConsole.retention_secs; + delete options.platform.drive.abci.tokioConsole.retention_secs; + } + + // move SSL files + if (options.network !== NETWORK_MAINNET) { + const filenames = ['private.key', 'bundle.crt', 'bundle.csr', 'csr.pem']; + + for (const filename of filenames) { + const oldFilePath = homeDir.joinPath( + name, + 'platform', + 'dapi', + 'envoy', + 'ssl', + filename, + ); + const newFilePath = homeDir.joinPath( + name, + 'platform', + 'gateway', + 'ssl', + filename, + ); + + if (fs.existsSync(oldFilePath)) { + fs.mkdirSync(path.dirname(newFilePath), { recursive: true }); + fs.copyFileSync(oldFilePath, newFilePath); + fs.rmSync(oldFilePath, { recursive: true }); + } + } + } }); return configFile; diff --git a/packages/dashmate/docker-compose.build.dapi_tx_filter_stream.yml b/packages/dashmate/docker-compose.build.dapi_core_streams.yml similarity index 96% rename from packages/dashmate/docker-compose.build.dapi_tx_filter_stream.yml rename to packages/dashmate/docker-compose.build.dapi_core_streams.yml index 35e82f593a..ec9cfdcf45 100644 --- a/packages/dashmate/docker-compose.build.dapi_tx_filter_stream.yml +++ b/packages/dashmate/docker-compose.build.dapi_core_streams.yml @@ -1,7 +1,7 @@ version: '3.7' services: - dapi_tx_filter_stream: + dapi_core_streams: build: context: ${PLATFORM_DAPI_API_DOCKER_BUILD_CONTEXT:?err} dockerfile: ${PLATFORM_DAPI_API_DOCKER_BUILD_DOCKER_FILE:?err} diff --git a/packages/dashmate/docker-compose.rate_limiter.metrics.yml b/packages/dashmate/docker-compose.rate_limiter.metrics.yml new file mode 100644 index 0000000000..7e76d9959c --- /dev/null +++ b/packages/dashmate/docker-compose.rate_limiter.metrics.yml @@ -0,0 +1,20 @@ +version: '3.7' + +services: + gateway_rate_limiter_metrics: + image: ${PLATFORM_GATEWAY_RATE_LIMITER_METRICS_DOCKER_IMAGE:?err} + labels: + org.dashmate.service.title: "Gateway rate limiter metrics exporter" + restart: unless-stopped + entrypoint: /bin/statsd_exporter + command: + - "--statsd.mapping-config=/etc/statsd-exporter/config.yaml" + networks: + - gateway_rate_limiter + volumes: + - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/gateway/rate_limiter/statsd_exporter.yaml:/etc/statsd-exporter/config.yaml:ro + expose: + - 9125 + - 9125/udp + ports: + - ${PLATFORM_GATEWAY_RATE_LIMITER_METRICS_HOST:?err}:${PLATFORM_GATEWAY_RATE_LIMITER_METRICS_PORT:?err}:9102 diff --git a/packages/dashmate/docker-compose.rate_limiter.yml b/packages/dashmate/docker-compose.rate_limiter.yml new file mode 100644 index 0000000000..9833a62036 --- /dev/null +++ b/packages/dashmate/docker-compose.rate_limiter.yml @@ -0,0 +1,54 @@ +version: '3.7' + +services: + gateway: + depends_on: + - dapi_api + - dapi_core_streams + - drive_abci + - gateway_rate_limiter + + gateway_rate_limiter: + labels: + org.dashmate.service.title: "Gateway rate limiter" + restart: unless-stopped + image: ${PLATFORM_GATEWAY_RATE_LIMITER_DOCKER_IMAGE:?err} + command: /bin/ratelimit + depends_on: + - gateway_rate_limiter_redis + networks: + - gateway_rate_limiter + volumes: + - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/gateway/rate_limiter/rate_limiter.yaml:/data/ratelimit/config/config.yaml:ro + environment: + - LOG_LEVEL=info + - LOG_FORMAT=text + - BACKEND_TYPE=redis + - REDIS_SOCKET_TYPE=tcp + - REDIS_URL=gateway_rate_limiter_redis:6379 + - RUNTIME_ROOT=/data + - RUNTIME_SUBDIRECTORY=ratelimit + - RUNTIME_WATCH_ROOT=false + - DISABLE_STATS=${PLATFORM_GATEWAY_RATE_LIMITER_METRICS_DISABLED:?err} + - STATSD_HOST=gateway_rate_limiter_metrics + - STATSD_PORT=9125 + - CONFIG_TYPE=FILE + - GRPC_MAX_CONNECTION_AGE=1h + - GRPC_MAX_CONNECTION_AGE_GRACE=10m + - GRPC_PORT=8081 + expose: + - 8081 + profiles: + - platform + + gateway_rate_limiter_redis: + labels: + org.dashmate.service.title: "Gateway rate limiter storage" + restart: unless-stopped + image: redis:alpine + expose: + - 6379 + networks: + - gateway_rate_limiter + profiles: + - platform diff --git a/packages/dashmate/docker-compose.yml b/packages/dashmate/docker-compose.yml index 9ee4f0c008..a2fad5c55a 100644 --- a/packages/dashmate/docker-compose.yml +++ b/packages/dashmate/docker-compose.yml @@ -24,8 +24,8 @@ services: org.dashmate.service.title: "Core" restart: unless-stopped ports: - - ${CORE_P2P_HOST:?err}:${CORE_P2P_PORT:?err}:${CORE_P2P_PORT:?err} # P2P - - ${CORE_RPC_HOST:?err}:${CORE_RPC_PORT:?err}:${CORE_RPC_PORT:?err} #RPC + - ${CORE_P2P_HOST:?err}:${CORE_P2P_PORT:?err}:${CORE_P2P_PORT:?err} + - ${CORE_RPC_HOST:?err}:${CORE_RPC_PORT:?err}:${CORE_RPC_PORT:?err} volumes: - core_data:/home/dash - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/core/dash.conf:/home/dash/.dashcore/dash.conf:ro @@ -72,16 +72,15 @@ services: - DB_PATH=/var/lib/dash/rs-drive-abci/db - ABCI_CONSENSUS_BIND_ADDRESS=tcp://0.0.0.0:26658 - GRPC_BIND_ADDRESS=0.0.0.0:26670 - - PROMETHEUS_BIND_ADDRESS=http://0.0.0.0:29090 + - PROMETHEUS_BIND_ADDRESS=${PLATFORM_DRIVE_ABCI_METRICS_URL} - TOKIO_CONSOLE_ENABLED=${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_ENABLED:?err} - TOKIO_CONSOLE_ADDRESS=0.0.0.0:${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_PORT:?err} - - TOKIO_CONSOLE_RETENTION_SECS=${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_RETENTION_SECS:?err} + - TOKIO_CONSOLE_RETENTION_SECS=${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_RETENTION:?err} stop_grace_period: 30s expose: - 26658 - 26659 - 26670 - - 29090 env_file: # Logger settings - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/drive/abci/logger.env @@ -89,6 +88,7 @@ services: - platform ports: - ${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_HOST:?err}:${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_PORT:?err}:${PLATFORM_DRIVE_ABCI_TOKIO_CONSOLE_PORT:?err} + - ${PLATFORM_DRIVE_ABCI_METRICS_HOST:?err}:${PLATFORM_DRIVE_ABCI_METRICS_PORT:?err}:29090 drive_tenderdash: image: ${PLATFORM_DRIVE_TENDERDASH_DOCKER_IMAGE:?err} @@ -140,13 +140,13 @@ services: - DRIVE_RPC_PORT=26670 command: yarn run api stop_grace_period: 10s - profiles: - - platform expose: - 3004 - 3005 + profiles: + - platform - dapi_tx_filter_stream: + dapi_core_streams: image: ${PLATFORM_DAPI_API_DOCKER_IMAGE:?err} labels: org.dashmate.service.title: "DAPI Transactions Filter Stream" @@ -170,30 +170,40 @@ services: - TENDERMINT_RPC_PORT=26657 - DRIVE_RPC_HOST=drive_abci - DRIVE_RPC_PORT=26670 + expose: + - 3006 command: yarn run core-streams stop_grace_period: 10s profiles: - platform - expose: - - 3006 - dapi_envoy: - image: ${PLATFORM_DAPI_ENVOY_DOCKER_IMAGE:?err} + gateway: + image: ${PLATFORM_GATEWAY_DOCKER_IMAGE:?err} labels: - org.dashmate.service.title: "DAPI Envoy" + org.dashmate.service.title: "Gateway" restart: unless-stopped ports: - - ${PLATFORM_DAPI_ENVOY_HTTP_HOST:?err}:${PLATFORM_DAPI_ENVOY_HTTP_PORT:?err}:10000 # JSON RPC and gRPC Web & Native + # HTTP entry point to the platform. + # Supports HTTP1 and HTTP2 + # Serves JSON RPC, gRPC, and gRPC-Web + - ${PLATFORM_GATEWAY_LISTENERS_DAPI_AND_DRIVE_HOST:?err}:${PLATFORM_GATEWAY_LISTENERS_DAPI_AND_DRIVE_PORT:?err}:10000 + - ${PLATFORM_GATEWAY_METRICS_HOST:?err}:${PLATFORM_GATEWAY_METRICS_PORT:?err}:9090 + - ${PLATFORM_GATEWAY_ADMIN_HOST:?err}:${PLATFORM_GATEWAY_ADMIN_PORT:?err}:9901 depends_on: - dapi_api - - dapi_tx_filter_stream + - dapi_core_streams + - drive_abci + networks: + - default + - gateway_rate_limiter environment: - ENVOY_UID=${LOCAL_UID:?err} - ENVOY_GID=${LOCAL_GID:?err} + - LOG_LEVEL=${PLATFORM_GATEWAY_LOG_LEVEL:?err} volumes: - - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/dapi/envoy/envoy.yaml:/etc/envoy/envoy.yaml:ro - - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/dapi/envoy/ssl/bundle.crt:/etc/ssl/bundle.crt:ro - - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/dapi/envoy/ssl/private.key:/etc/ssl/private.key:ro + - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/gateway/envoy.yaml:/etc/envoy/envoy.yaml:ro + - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/gateway/ssl/bundle.crt:/etc/ssl/bundle.crt:ro + - ${DASHMATE_HOME_DIR:?err}/${CONFIG_NAME:?err}/platform/gateway/ssl/private.key:/etc/ssl/private.key:ro stop_grace_period: 10s profiles: - platform @@ -205,6 +215,9 @@ volumes: networks: default: + driver: bridge ipam: config: - subnet: ${DOCKER_NETWORK_SUBNET:?err} + gateway_rate_limiter: + driver: bridge diff --git a/packages/dashmate/scripts/helper.js b/packages/dashmate/scripts/helper.js index 5ed920107c..6cd9feb9f6 100644 --- a/packages/dashmate/scripts/helper.js +++ b/packages/dashmate/scripts/helper.js @@ -47,8 +47,8 @@ import createDIContainer from '../src/createDIContainer.js'; configFile: asValue(configFile), }); - const provider = config.get('platform.dapi.envoy.ssl.provider'); - const isEnabled = config.get('platform.dapi.envoy.ssl.enabled'); + const provider = config.get('platform.gateway.ssl.provider'); + const isEnabled = config.get('platform.gateway.ssl.enabled'); if (isEnabled && provider === 'zerossl') { const scheduleRenewZeroSslCertificate = container.resolve('scheduleRenewZeroSslCertificate'); diff --git a/packages/dashmate/src/commands/config/render.js b/packages/dashmate/src/commands/config/render.js index 76a2f117db..0e9c58ae0b 100644 --- a/packages/dashmate/src/commands/config/render.js +++ b/packages/dashmate/src/commands/config/render.js @@ -31,6 +31,6 @@ Force dashmate to render all config's service configs writeServiceConfigs(config.getName(), configFiles); // eslint-disable-next-line no-console - console.log(`Config "${config.getName()}" service configs rendered`); + console.log(`"${config.getName()}" service configs rendered`); } } diff --git a/packages/dashmate/src/config/configJsonSchema.js b/packages/dashmate/src/config/configJsonSchema.js index 357f5e5ecf..b003a4aded 100644 --- a/packages/dashmate/src/config/configJsonSchema.js +++ b/packages/dashmate/src/config/configJsonSchema.js @@ -50,6 +50,11 @@ export default { required: ['image', 'build'], additionalProperties: false, }, + host: { + type: 'string', + minLength: 1, + format: 'ipv4', + }, port: { type: 'integer', minimum: 0, @@ -84,6 +89,22 @@ export default { type: 'string', pattern: '^[0-9]+(\\.[0-9]+)?s$', }, + enabledHostPort: { + type: 'object', + properties: { + enabled: { + type: 'boolean', + }, + host: { + $ref: '#/definitions/host', + }, + port: { + $ref: '#/definitions/port', + }, + }, + additionalProperties: false, + required: ['enabled', 'host', 'port'], + }, }, properties: { description: { @@ -173,9 +194,7 @@ export default { type: 'object', properties: { host: { - type: 'string', - minLength: 1, - format: 'ipv4', + $ref: '#/definitions/host', }, port: { $ref: '#/definitions/port', @@ -347,18 +366,75 @@ export default { platform: { type: 'object', properties: { - dapi: { + gateway: { type: 'object', properties: { - envoy: { + docker: { + $ref: '#/definitions/docker', + }, + maxConnections: { + type: 'integer', + minimum: 1, + description: 'Maximum number of connections that Gateway accepts from downstream clients', + }, + maxHeapSizeInBytes: { + type: 'integer', + minimum: 1, + description: 'Maximum heap size in bytes. If the heap size exceeds this value, Gateway will take actions to reduce memory usage', + }, + upstreams: { type: 'object', properties: { - docker: { - $ref: '#/definitions/docker', + driveGrpc: { + $id: 'gatewayUpstream', + type: 'object', + properties: { + maxRequests: { + type: 'integer', + minimum: 1, + description: 'The maximum number of parallel requests', + }, + }, + required: ['maxRequests'], + additionalProperties: false, + }, + dapiApi: { + $ref: 'gatewayUpstream', + }, + dapiCoreStreams: { + $ref: 'gatewayUpstream', }, - http: { + dapiJsonRpc: { + $ref: 'gatewayUpstream', + }, + }, + additionalProperties: false, + required: ['driveGrpc', 'dapiApi', 'dapiCoreStreams', 'dapiJsonRpc'], + }, + metrics: { + $ref: '#/definitions/enabledHostPort', + }, + admin: { + $ref: '#/definitions/enabledHostPort', + }, + listeners: { + type: 'object', + properties: { + dapiAndDrive: { type: 'object', properties: { + http2: { + type: 'object', + properties: { + maxConcurrentStreams: { + type: 'integer', + minimum: 1, + description: 'Maximum number of concurrent streams allowed for each connection', + }, + }, + additionalProperties: false, + required: ['maxConcurrentStreams'], + }, host: { type: 'string', minLength: 1, @@ -367,75 +443,228 @@ export default { port: { $ref: '#/definitions/port', }, - connectTimeout: { - $ref: '#/definitions/durationInSeconds', - }, - responseTimeout: { - $ref: '#/definitions/durationInSeconds', - }, }, - required: ['host', 'port', 'connectTimeout', 'responseTimeout'], + required: ['http2', 'host', 'port'], additionalProperties: false, }, - rateLimiter: { + }, + required: ['dapiAndDrive'], + additionalProperties: false, + }, + rateLimiter: { + type: 'object', + properties: { + docker: { + $ref: '#/definitions/docker', + }, + unit: { + type: 'string', + enum: ['second', 'minute', 'hour', 'day'], + }, + requestsPerUnit: { + type: 'integer', + minimum: 1, + }, + blacklist: { + type: 'array', + items: { + $ref: '#/definitions/host', + }, + description: 'List of IP addresses that are blacklisted from making requests', + }, + whitelist: { + type: 'array', + items: { + $ref: '#/definitions/host', + }, + description: 'List of IP addresses that are whitelisted to make requests without limits', + }, + metrics: { type: 'object', properties: { - maxTokens: { - type: 'integer', - minimum: 0, - }, - tokensPerFill: { - type: 'integer', - minimum: 0, - }, - fillInterval: { - $ref: '#/definitions/duration', + docker: { + $ref: '#/definitions/docker', }, enabled: { type: 'boolean', }, + host: { + $ref: '#/definitions/host', + }, + port: { + $ref: '#/definitions/port', + }, }, - required: ['enabled', 'fillInterval', 'tokensPerFill', 'maxTokens'], additionalProperties: false, + required: ['docker', 'enabled', 'host', 'port'], }, - ssl: { + enabled: { + type: 'boolean', + }, + }, + required: ['docker', 'enabled', 'unit', 'requestsPerUnit', 'blacklist', 'whitelist', 'metrics'], + additionalProperties: false, + }, + ssl: { + type: 'object', + properties: { + enabled: { + type: 'boolean', + }, + provider: { + type: 'string', + enum: ['zerossl', 'self-signed', 'file'], + }, + providerConfigs: { type: 'object', properties: { - enabled: { - type: 'boolean', - }, - provider: { - type: 'string', - enum: ['zerossl', 'self-signed', 'file'], - }, - providerConfigs: { - type: 'object', + zerossl: { + type: ['object'], properties: { - zerossl: { - type: ['object'], + apiKey: { + type: ['string', 'null'], + minLength: 32, + }, + id: { + type: ['string', 'null'], + minLength: 32, + }, + }, + required: ['apiKey', 'id'], + additionalProperties: false, + }, + }, + }, + }, + required: ['provider', 'providerConfigs', 'enabled'], + additionalProperties: false, + }, + log: { + type: 'object', + properties: { + level: { + type: 'string', + enum: ['trace', 'debug', 'info', 'warn', 'error', 'critical', 'off'], + }, + accessLogs: { + type: 'array', + items: { + oneOf: [ + { + type: 'object', + properties: { + type: { + type: 'string', + minLength: 1, + enum: ['stdout', 'stderr'], + description: 'Access log type: stdout, stderr or file', + }, + format: { + type: 'string', + enum: ['text', 'json'], + }, + template: true, + }, + required: ['type', 'format'], + additionalProperties: false, + if: { + type: 'object', properties: { - apiKey: { - type: ['string', 'null'], - minLength: 32, + format: { + const: 'json', }, - id: { - type: ['string', 'null'], - minLength: 32, + }, + }, + then: { + type: 'object', + properties: { + template: { + type: ['null', 'object'], + additionalProperties: { + type: 'string', + }, + description: 'JSON fields and values. If null, default template is used.', }, }, - required: ['apiKey', 'id'], - additionalProperties: false, + required: ['template'], + }, + else: { + type: 'object', + properties: { + template: { + type: ['null', 'string'], + description: 'Template string. If null, default template is used.', + }, + }, + required: ['template'], }, }, - }, + { + type: 'object', + properties: { + type: { + type: 'string', + const: 'file', + description: 'Access log type: stdout, stderr or file', + }, + format: { + type: 'string', + enum: ['text', 'json'], + }, + path: { + type: 'string', + minLength: 1, + }, + template: true, + }, + required: ['type', 'format', 'path'], + additionalProperties: false, + if: { + type: 'object', + properties: { + format: { + const: 'json', + }, + }, + }, + then: { + type: 'object', + properties: { + template: { + type: ['null', 'object'], + additionalProperties: { + type: 'string', + }, + description: 'JSON fields and values. If null, default template is used.', + }, + }, + required: ['template'], + }, + else: { + type: 'object', + properties: { + template: { + type: ['null', 'string'], + description: 'Template string. If null, default template is used.', + }, + }, + required: ['template'], + }, + }, + ], }, - required: ['provider', 'providerConfigs', 'enabled'], - additionalProperties: false, }, }, - required: ['docker', 'http', 'rateLimiter', 'ssl'], additionalProperties: false, + required: ['level', 'accessLogs'], }, + }, + required: ['docker', 'listeners', 'rateLimiter', 'ssl', 'maxHeapSizeInBytes', 'maxConnections', 'upstreams', 'metrics', 'admin', 'log'], + additionalProperties: false, + }, + dapi: { + type: 'object', + properties: { api: { type: 'object', properties: { @@ -469,7 +698,7 @@ export default { additionalProperties: false, }, }, - required: ['envoy', 'api'], + required: ['api'], additionalProperties: false, }, drive: { @@ -520,19 +749,18 @@ export default { type: 'boolean', }, host: { - type: 'string', - minLength: 1, - format: 'ipv4', + $ref: '#/definitions/host', }, port: { $ref: '#/definitions/port', }, - retention_secs: { + retention: { type: 'integer', minimum: 0, + description: 'How many seconds keep data if console is not connected', }, }, - required: ['enabled', 'host', 'port', 'retention_secs'], + required: ['enabled', 'host', 'port', 'retention'], additionalProperties: false, }, validatorSet: { @@ -571,9 +799,12 @@ export default { type: 'integer', minimum: 180, }, + metrics: { + $ref: '#/definitions/enabledHostPort', + }, }, additionalProperties: false, - required: ['docker', 'logs', 'tokioConsole', 'validatorSet', 'chainLock', 'epochTime'], + required: ['docker', 'logs', 'tokioConsole', 'validatorSet', 'chainLock', 'epochTime', 'metrics'], }, tenderdash: { type: 'object', @@ -791,23 +1022,7 @@ export default { additionalProperties: false, }, metrics: { - description: 'Prometheus metrics', - type: 'object', - properties: { - enabled: { - type: 'boolean', - }, - host: { - type: 'string', - minLength: 1, - format: 'ipv4', - }, - port: { - $ref: '#/definitions/port', - }, - }, - required: ['enabled', 'host', 'port'], - additionalProperties: false, + $ref: '#/definitions/enabledHostPort', }, node: { type: 'object', @@ -981,7 +1196,7 @@ export default { type: 'boolean', }, }, - required: ['dapi', 'drive', 'dpns', 'dashpay', 'featureFlags', 'sourcePath', 'masternodeRewardShares', 'withdrawals', 'enable'], + required: ['gateway', 'dapi', 'drive', 'dpns', 'dashpay', 'featureFlags', 'sourcePath', 'masternodeRewardShares', 'withdrawals', 'enable'], additionalProperties: false, }, dashmate: { diff --git a/packages/dashmate/src/config/generateEnvsFactory.js b/packages/dashmate/src/config/generateEnvsFactory.js index 9aedfbb95d..fcb0491ecf 100644 --- a/packages/dashmate/src/config/generateEnvsFactory.js +++ b/packages/dashmate/src/config/generateEnvsFactory.js @@ -44,7 +44,7 @@ export default function generateEnvsFactory(configFile, homeDir, getConfigProfil if (config.get('platform.dapi.api.docker.build.enabled')) { dockerComposeFiles.push('docker-compose.build.dapi_api.yml'); - dockerComposeFiles.push('docker-compose.build.dapi_tx_filter_stream.yml'); + dockerComposeFiles.push('docker-compose.build.dapi_core_streams.yml'); } } @@ -56,6 +56,14 @@ export default function generateEnvsFactory(configFile, homeDir, getConfigProfil dockerComposeFiles.push(insightComposeFile); } + if (config.get('platform.gateway.rateLimiter.enabled')) { + dockerComposeFiles.push('docker-compose.rate_limiter.yml'); + + if (config.get('platform.gateway.rateLimiter.metrics.enabled')) { + dockerComposeFiles.push('docker-compose.rate_limiter.metrics.yml'); + } + } + // we need this for compatibility with old configs const projectIdWithPrefix = configFile.getProjectId() ? `_${configFile.getProjectId()}` : ''; @@ -68,6 +76,11 @@ export default function generateEnvsFactory(configFile, homeDir, getConfigProfil tenderdashLogDirectoryPath = path.dirname(tenderdashLogFilePath); } + let driveAbciMetricsUrl = ''; + if (config.get('platform.drive.abci.metrics.enabled')) { + driveAbciMetricsUrl = 'http://0.0.0.0:29090'; + } + return { DASHMATE_HOME_DIR: homeDir.getPath(), LOCAL_UID: uid, @@ -84,6 +97,8 @@ export default function generateEnvsFactory(configFile, homeDir, getConfigProfil ), DASHMATE_HELPER_DOCKER_IMAGE, PLATFORM_DRIVE_TENDERDASH_LOG_DIRECTORY_PATH: tenderdashLogDirectoryPath, + PLATFORM_GATEWAY_RATE_LIMITER_METRICS_DISABLED: !config.get('platform.gateway.rateLimiter.metrics.enabled'), + PLATFORM_DRIVE_ABCI_METRICS_URL: driveAbciMetricsUrl, ...convertObjectToEnvs(config.getOptions()), }; } diff --git a/packages/dashmate/src/core/wallet/registerMasternode.js b/packages/dashmate/src/core/wallet/registerMasternode.js index d5d2dba275..285c65409b 100644 --- a/packages/dashmate/src/core/wallet/registerMasternode.js +++ b/packages/dashmate/src/core/wallet/registerMasternode.js @@ -46,7 +46,7 @@ export default async function registerMasternode( if (hp) { const platformNodeId = config.get('platform.drive.tenderdash.node.id'); const platformP2PPort = config.get('platform.drive.tenderdash.p2p.port'); - const platformHttpPort = config.get('platform.dapi.envoy.http.port'); + const platformHttpPort = config.get('platform.gateway.listeners.dapiAndDrive.port'); proTxArgs.push(platformNodeId); proTxArgs.push(platformP2PPort.toString()); diff --git a/packages/dashmate/src/helper/scheduleRenewZeroSslCertificateFactory.js b/packages/dashmate/src/helper/scheduleRenewZeroSslCertificateFactory.js index a594315ef1..8b490c6b3e 100644 --- a/packages/dashmate/src/helper/scheduleRenewZeroSslCertificateFactory.js +++ b/packages/dashmate/src/helper/scheduleRenewZeroSslCertificateFactory.js @@ -26,8 +26,8 @@ export default function scheduleRenewZeroSslCertificateFactory( */ async function scheduleRenewZeroSslCertificate(config) { const certificate = await getCertificate( - config.get('platform.dapi.envoy.ssl.providerConfigs.zerossl.apiKey', false), - config.get('platform.dapi.envoy.ssl.providerConfigs.zerossl.id', false), + config.get('platform.gateway.ssl.providerConfigs.zerossl.apiKey', false), + config.get('platform.gateway.ssl.providerConfigs.zerossl.id', false), ); if (!certificate) { @@ -61,8 +61,11 @@ export default function scheduleRenewZeroSslCertificateFactory( configFileRepository.write(configFile); writeConfigTemplates(config); - // Restart Envoy to catch up new SSL certificates - await dockerCompose.execCommand(config, 'dapi_envoy', 'kill -SIGHUP 1'); + // TODO: We can use https://www.envoyproxy.io/docs/envoy/v1.30.1/start/quick-start/configuration-dynamic-filesystem.html#start-quick-start-dynamic-fs-dynamic-lds + // to dynamically update envoy configuration without restarting it + + // Restart Gateway to catch up new SSL certificates + await dockerCompose.execCommand(config, 'gateway', 'kill -SIGHUP 1'); return job.stop(); }, async () => { diff --git a/packages/dashmate/src/listr/prompts/createIpAndPortsForm.js b/packages/dashmate/src/listr/prompts/createIpAndPortsForm.js index 1a2495b8f3..ac93be1a62 100644 --- a/packages/dashmate/src/listr/prompts/createIpAndPortsForm.js +++ b/packages/dashmate/src/listr/prompts/createIpAndPortsForm.js @@ -107,7 +107,7 @@ export default function createIpAndPortsFormFactory(defaultConfigs) { if (initialPlatformHTTPPort === null || initialPlatformHTTPPort === undefined || network === PRESET_MAINNET) { - initialPlatformHTTPPort = defaultConfigs.get(network).get('platform.dapi.envoy.http.port').toString(); + initialPlatformHTTPPort = defaultConfigs.get(network).get('platform.gateway.listeners.dapiAndDrive.port').toString(); } fields.push({ diff --git a/packages/dashmate/src/listr/tasks/platform/waitForNodeToBeReadyTaskFactory.js b/packages/dashmate/src/listr/tasks/platform/waitForNodeToBeReadyTaskFactory.js index 3b1bb0c61c..7953d4bcd1 100644 --- a/packages/dashmate/src/listr/tasks/platform/waitForNodeToBeReadyTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/platform/waitForNodeToBeReadyTaskFactory.js @@ -19,8 +19,8 @@ export default function waitForNodeToBeReadyTaskFactory() { { title: `Wait for node ${config.getName()} to be ready`, task: async () => { - let host = config.get('platform.dapi.envoy.http.host'); - const port = config.get('platform.dapi.envoy.http.port'); + let host = config.get('platform.gateway.listeners.dapiAndDrive.host'); + const port = config.get('platform.gateway.listeners.dapiAndDrive.port'); if (host === '0.0.0.0') { host = '127.0.0.1'; diff --git a/packages/dashmate/src/listr/tasks/setup/regular/configureNodeTaskFactory.js b/packages/dashmate/src/listr/tasks/setup/regular/configureNodeTaskFactory.js index bdcbc554d8..3c227b1d50 100644 --- a/packages/dashmate/src/listr/tasks/setup/regular/configureNodeTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/setup/regular/configureNodeTaskFactory.js @@ -86,7 +86,7 @@ export default function configureNodeTaskFactory(createIpAndPortsForm) { ctx.config.set('core.p2p.port', form.coreP2PPort); if (ctx.isHP) { - ctx.config.set('platform.dapi.envoy.http.port', form.platformHTTPPort); + ctx.config.set('platform.gateway.listeners.dapiAndDrive.port', form.platformHTTPPort); ctx.config.set('platform.drive.tenderdash.p2p.port', form.platformP2PPort); } } diff --git a/packages/dashmate/src/listr/tasks/setup/regular/configureSSLCertificateTaskFactory.js b/packages/dashmate/src/listr/tasks/setup/regular/configureSSLCertificateTaskFactory.js index f489be8de3..62c9f154d0 100644 --- a/packages/dashmate/src/listr/tasks/setup/regular/configureSSLCertificateTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/setup/regular/configureSSLCertificateTaskFactory.js @@ -91,7 +91,7 @@ export default function configureSSLCertificateTaskFactory( }, }); - ctx.config.set('platform.dapi.envoy.ssl.providerConfigs.zerossl.apiKey', apiKey); + ctx.config.set('platform.gateway.ssl.providerConfigs.zerossl.apiKey', apiKey); return obtainZeroSSLCertificateTask(ctx.config); }, @@ -141,7 +141,7 @@ export default function configureSSLCertificateTaskFactory( }); } - ctx.config.set('platform.dapi.envoy.ssl.provider', ctx.certificateProvider); + ctx.config.set('platform.gateway.ssl.provider', ctx.certificateProvider); // eslint-disable-next-line no-param-reassign task.output = ctx.certificateProvider; diff --git a/packages/dashmate/src/listr/tasks/setup/regular/getConfigurationOutputFromContext.js b/packages/dashmate/src/listr/tasks/setup/regular/getConfigurationOutputFromContext.js index de7f2be963..d9cb36a9cb 100644 --- a/packages/dashmate/src/listr/tasks/setup/regular/getConfigurationOutputFromContext.js +++ b/packages/dashmate/src/listr/tasks/setup/regular/getConfigurationOutputFromContext.js @@ -23,7 +23,7 @@ export default async function getConfigurationOutputFromContext(ctx) { if (ctx.isHP) { output += `\n\nPlatform P2P port: ${ctx.config.get('platform.drive.tenderdash.p2p.port')} - Platform HTTP port: ${ctx.config.get('platform.dapi.envoy.http.port')}`; + Platform HTTP port: ${ctx.config.get('platform.gateway.listeners.dapiAndDrive.port')}`; } return output; diff --git a/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithCoreWallet.js b/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithCoreWallet.js index 8688af05bd..dda9811bcf 100644 --- a/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithCoreWallet.js +++ b/packages/dashmate/src/listr/tasks/setup/regular/registerMasternode/registerMasternodeWithCoreWallet.js @@ -218,7 +218,7 @@ export default function registerMasternodeWithCoreWalletFactory(createIpAndPorts const platformHTTPPort = state.ipAndPorts.platformHTTPPort || defaultConfigs.get(ctx.preset) - .get('platform.dapi.envoy.http.port'); + .get('platform.gateway.listeners.dapiAndDrive.port'); let command; if (ctx.isHP) { diff --git a/packages/dashmate/src/listr/tasks/setup/regular/registerMasternodeGuideTaskFactory.js b/packages/dashmate/src/listr/tasks/setup/regular/registerMasternodeGuideTaskFactory.js index b65c2c84bf..ce358f56fd 100644 --- a/packages/dashmate/src/listr/tasks/setup/regular/registerMasternodeGuideTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/setup/regular/registerMasternodeGuideTaskFactory.js @@ -73,7 +73,7 @@ export default function registerMasternodeGuideTaskFactory( ctx.config.set('platform.drive.tenderdash.node.id', deriveTenderdashNodeId(state.platformNodeKey)); ctx.config.set('platform.drive.tenderdash.node.key', state.platformNodeKey); - ctx.config.set('platform.dapi.envoy.http.port', state.ipAndPorts.platformHTTPPort); + ctx.config.set('platform.gateway.listeners.dapiAndDrive.port', state.ipAndPorts.platformHTTPPort); ctx.config.set('platform.drive.tenderdash.p2p.port', state.ipAndPorts.platformP2PPort); } diff --git a/packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js b/packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js index 70a8cd7f96..51e9918a06 100644 --- a/packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js @@ -223,7 +223,11 @@ export default function setupLocalPresetTaskFactory( config.set('platform.drive.tenderdash.node.key', key); config.set('platform.drive.abci.tokioConsole.port', config.get('platform.drive.abci.tokioConsole.port') + (i * 100)); - config.set('platform.dapi.envoy.http.port', config.get('platform.dapi.envoy.http.port') + (i * 100)); + config.set('platform.drive.abci.metrics.port', config.get('platform.drive.abci.metrics.port') + (i * 100)); + config.set('platform.gateway.admin.port', config.get('platform.gateway.admin.port') + (i * 100)); + config.set('platform.gateway.listeners.dapiAndDrive.port', config.get('platform.gateway.listeners.dapiAndDrive.port') + (i * 100)); + config.set('platform.gateway.metrics.port', config.get('platform.gateway.metrics.port') + (i * 100)); + config.set('platform.gateway.rateLimiter.metrics.port', config.get('platform.gateway.rateLimiter.metrics.port') + (i * 100)); config.set('platform.drive.tenderdash.p2p.port', config.get('platform.drive.tenderdash.p2p.port') + (i * 100)); config.set('platform.drive.tenderdash.rpc.port', config.get('platform.drive.tenderdash.rpc.port') + (i * 100)); config.set('platform.drive.tenderdash.pprof.port', config.get('platform.drive.tenderdash.pprof.port') + (i * 100)); diff --git a/packages/dashmate/src/listr/tasks/ssl/VerificationServer.js b/packages/dashmate/src/listr/tasks/ssl/VerificationServer.js index 738c427611..eba5de00e4 100644 --- a/packages/dashmate/src/listr/tasks/ssl/VerificationServer.js +++ b/packages/dashmate/src/listr/tasks/ssl/VerificationServer.js @@ -39,8 +39,8 @@ export default class VerificationServer { dots.templateSettings.strip = false; - // Set up Envoy config - const configSubPath = path.join('platform', 'dapi', 'envoy'); + // Set up Gateway config + const configSubPath = path.join('platform', 'gateway'); const templatePath = path.join(TEMPLATES_DIR, configSubPath, '_zerossl_validation.yaml.dot'); const templateString = fs.readFileSync(templatePath, 'utf-8'); const template = dots.template(templateString); @@ -48,9 +48,9 @@ export default class VerificationServer { const route = validationUrl.replace(`http://${config.get('externalIp')}`, ''); const body = validationContent.join('\\n'); - const envoyConfig = template({ route, body }); + const gatewayConfig = template({ route, body }); - const configDir = this.homeDir.joinPath(config.getName(), 'platform', 'dapi', 'envoy'); + const configDir = this.homeDir.joinPath(config.getName(), 'platform', 'gateway'); const configName = path.basename(templatePath, '.dot'); this.configPath = path.join(configDir, configName); @@ -59,7 +59,7 @@ export default class VerificationServer { fs.mkdirSync(configDir); } fs.rmSync(this.configPath, { force: true }); - fs.writeFileSync(this.configPath, envoyConfig, 'utf8'); + fs.writeFileSync(this.configPath, gatewayConfig, 'utf8'); } /** @@ -76,7 +76,7 @@ export default class VerificationServer { return false; } - const image = this.config.get('platform.dapi.envoy.docker.image'); + const image = this.config.get('platform.gateway.docker.image'); const name = 'dashmate-zerossl-validation'; diff --git a/packages/dashmate/src/listr/tasks/ssl/saveCertificateTask.js b/packages/dashmate/src/listr/tasks/ssl/saveCertificateTask.js index b3096936c4..bbcfac1ea0 100644 --- a/packages/dashmate/src/listr/tasks/ssl/saveCertificateTask.js +++ b/packages/dashmate/src/listr/tasks/ssl/saveCertificateTask.js @@ -20,8 +20,7 @@ export default function saveCertificateTaskFactory(homeDir) { const certificatesDir = homeDir.joinPath( config.getName(), 'platform', - 'dapi', - 'envoy', + 'gateway', 'ssl', ); @@ -34,7 +33,7 @@ export default function saveCertificateTaskFactory(homeDir) { const keyFile = path.join(certificatesDir, 'private.key'); fs.writeFileSync(keyFile, ctx.privateKeyFile, 'utf8'); - config.set('platform.dapi.envoy.ssl.enabled', true); + config.set('platform.gateway.ssl.enabled', true); }, }]); } diff --git a/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js b/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js index 16e5c66933..12718ad4de 100644 --- a/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/ssl/zerossl/obtainZeroSSLCertificateTaskFactory.js @@ -37,10 +37,10 @@ export default function obtainZeroSSLCertificateTaskFactory( */ async function obtainZeroSSLCertificateTask(config) { // Make sure that required config options are set - const apiKey = config.get('platform.dapi.envoy.ssl.providerConfigs.zerossl.apiKey', true); + const apiKey = config.get('platform.gateway.ssl.providerConfigs.zerossl.apiKey', true); const externalIp = config.get('externalIp', true); - const sslConfigDir = homeDir.joinPath(config.getName(), 'platform', 'dapi', 'envoy', 'ssl'); + const sslConfigDir = homeDir.joinPath(config.getName(), 'platform', 'gateway', 'ssl'); const csrFilePath = path.join(sslConfigDir, 'csr.pem'); const privateKeyFilePath = path.join(sslConfigDir, 'private.key'); const bundleFilePath = path.join(sslConfigDir, 'bundle.crt'); @@ -54,7 +54,7 @@ export default function obtainZeroSSLCertificateTaskFactory( // Skips the check if force flag is set skip: (ctx) => ctx.force, task: async (ctx, task) => { - const certificateId = await config.get('platform.dapi.envoy.ssl.providerConfigs.zerossl.id'); + const certificateId = await config.get('platform.gateway.ssl.providerConfigs.zerossl.id'); if (!certificateId) { // Certificate is not configured @@ -179,9 +179,9 @@ export default function obtainZeroSSLCertificateTaskFactory( apiKey, ); - config.set('platform.dapi.envoy.ssl.enabled', true); - config.set('platform.dapi.envoy.ssl.provider', 'zerossl'); - config.set('platform.dapi.envoy.ssl.providerConfigs.zerossl.id', ctx.certificate.id); + config.set('platform.gateway.ssl.enabled', true); + config.set('platform.gateway.ssl.provider', 'zerossl'); + config.set('platform.gateway.ssl.providerConfigs.zerossl.id', ctx.certificate.id); }, }, { diff --git a/packages/dashmate/src/listr/tasks/startNodeTaskFactory.js b/packages/dashmate/src/listr/tasks/startNodeTaskFactory.js index ac2ea87a0c..9c0308b28c 100644 --- a/packages/dashmate/src/listr/tasks/startNodeTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/startNodeTaskFactory.js @@ -45,16 +45,22 @@ export default function startNodeTaskFactory( // Check Drive log files are created if (config.get('platform.enable')) { + // Ensure log files for Drive are created const loggers = config.get('platform.drive.abci.logs'); - - for (const logger of Object.values(loggers)) { - if (['stdout', 'stderr'].includes(logger.destination)) { - continue; - } - - ensureFileMountExists(logger.destination, 0o666); - } - + Object.values(loggers) + .filter((logger) => logger.destination !== 'stdout' && logger.destination !== 'stderr') + .forEach((logger) => { + ensureFileMountExists(logger.destination, 0o666); + }); + + // Ensure access log files for Gateway are created + config.get('platform.gateway.log.accessLogs') + .filter((log) => log.type === 'file') + .forEach((log) => { + ensureFileMountExists(log.path, 0o666); + }); + + // Ensure tenderdash log file is created const tenderdashLogFilePath = config.get('platform.drive.tenderdash.log.path'); if (tenderdashLogFilePath !== null) { ensureFileMountExists(tenderdashLogFilePath, 0o666); diff --git a/packages/dashmate/src/status/scopes/platform.js b/packages/dashmate/src/status/scopes/platform.js index ad640a3076..b826775229 100644 --- a/packages/dashmate/src/status/scopes/platform.js +++ b/packages/dashmate/src/status/scopes/platform.js @@ -77,7 +77,7 @@ export default function getPlatformScopeFactory( // Collecting platform data fails if Tenderdash is waiting for core to sync if (info.serviceStatus === ServiceStatusEnum.up) { const portStatusResult = await Promise.allSettled([ - providers.mnowatch.checkPortStatus(config.get('platform.dapi.envoy.http.port')), + providers.mnowatch.checkPortStatus(config.get('platform.gateway.listeners.dapiAndDrive.port')), providers.mnowatch.checkPortStatus(config.get('platform.drive.tenderdash.p2p.port')), ]); const [httpPortState, p2pPortState] = portStatusResult.map((result) => (result.status === 'fulfilled' ? result.value : null)); @@ -185,7 +185,7 @@ export default function getPlatformScopeFactory( * @returns {Promise} */ async function getPlatformScope(config) { - const httpPort = config.get('platform.dapi.envoy.http.port'); + const httpPort = config.get('platform.gateway.listeners.dapiAndDrive.port'); const httpService = config.get('externalIp') ? `${config.get('externalIp')}:${httpPort}` : null; const p2pPort = config.get('platform.drive.tenderdash.p2p.port'); const p2pService = config.get('externalIp') ? `${config.get('externalIp')}:${p2pPort}` : null; diff --git a/packages/dashmate/src/test/constants/services.js b/packages/dashmate/src/test/constants/services.js index 1b29357cbb..beb3d1d9aa 100644 --- a/packages/dashmate/src/test/constants/services.js +++ b/packages/dashmate/src/test/constants/services.js @@ -1,9 +1,9 @@ export default { dashmate_helper: 'Dashmate Helper', - dapi_envoy: 'DAPI Envoy', + gateway: 'Gateway', dapi_api: 'DAPI API', drive_tenderdash: 'Drive Tenderdash', drive_abci: 'Drive ABCI', - dapi_tx_filter_stream: 'DAPI Transactions Filter Stream', + dapi_core_streams: 'DAPI Core Streams', core: 'Core', }; diff --git a/packages/dashmate/src/test/mock/getConfigMock.js b/packages/dashmate/src/test/mock/getConfigMock.js index 0e30ce5e0b..1961c48f28 100644 --- a/packages/dashmate/src/test/mock/getConfigMock.js +++ b/packages/dashmate/src/test/mock/getConfigMock.js @@ -9,11 +9,11 @@ export default function getConfigMock(sinon) { configMock.get.withArgs('core.rpc.host').returns('127.0.0.1'); configMock.get.withArgs('docker.network.privateInterface').returns('127.0.0.1'); configMock.get.withArgs('docker.network.privateInterface').returns('127.0.0.1'); - configMock.get.withArgs('platform.dapi.envoy.http.port').returns('8100'); + configMock.get.withArgs('platform.gateway.listeners.dapiAndDrive.port').returns('8100'); configMock.get.withArgs('externalIp').returns('127.0.0.1'); configMock.get.withArgs('platform.drive.tenderdash.p2p.port').returns('8101'); - configMock.get.withArgs('platform.dapi.envoy.http.host').returns('0.0.0.0'); - configMock.get.withArgs('platform.dapi.envoy.http.port').returns('8102'); + configMock.get.withArgs('platform.gateway.listeners.dapiAndDrive.host').returns('0.0.0.0'); + configMock.get.withArgs('platform.gateway.listeners.dapiAndDrive.port').returns('8102'); configMock.get.withArgs('platform.drive.tenderdash.rpc.host').returns('127.0.0.1'); configMock.get.withArgs('platform.drive.tenderdash.rpc.port').returns('8103'); configMock.get.withArgs('platform.enable').returns(true); diff --git a/packages/dashmate/templates/dynamic-compose.yml.dot b/packages/dashmate/templates/dynamic-compose.yml.dot index 84fed7fa04..c0055e6949 100644 --- a/packages/dashmate/templates/dynamic-compose.yml.dot +++ b/packages/dashmate/templates/dynamic-compose.yml.dot @@ -17,3 +17,12 @@ services: - {{=settings.destination}}:/var/log/dash/drive/{{=name}}/{{=settings.destination.split('/').reverse()[0]}} {{~}} {{?}} + + {{ gatewayLogs = it.platform.gateway.log.accessLogs.filter((l) => l.type === 'file'); }} + {{? gatewayLogs.length > 0 }} + gateway: + volumes: + {{~ gatewayLogs :log }} + - {{= log.path }}:/var/log/{{= log.path.split('/').reverse()[0] }} + {{~}} + {{?}} diff --git a/packages/dashmate/templates/platform/dapi/envoy/envoy.yaml.dot b/packages/dashmate/templates/platform/dapi/envoy/envoy.yaml.dot deleted file mode 100644 index dab78d565f..0000000000 --- a/packages/dashmate/templates/platform/dapi/envoy/envoy.yaml.dot +++ /dev/null @@ -1,263 +0,0 @@ -!ignore filters: &filters - - name: envoy.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress_http - codec_type: auto - access_log: - - name: envoy.access_loggers.file - typed_config: - "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog - path: /dev/stdout - log_format: - json_format: - timestamp: "%START_TIME%" - client: "%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%" - protocol: "%PROTOCOL%" - method: "%REQ(:METHOD)%" - uri: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" - upstream: "%UPSTREAM_HOST%" - "http-status": "%RESPONSE_CODE%" - "grpc-status": "%GRPC_STATUS%" - "rx-bytes": "%BYTES_RECEIVED%" - "tx-bytes": "%BYTES_SENT%" - "response-flags": "%RESPONSE_FLAGS%" - duration: "%DURATION%" - authority: "%REQ(:AUTHORITY)%" - http_filters: - - name: envoy.filters.http.local_ratelimit - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.local_ratelimit.v3.LocalRateLimit - stat_prefix: http_local_rate_limiter - # see documentation https://www.envoyproxy.io/docs/envoy/latest/api-v3/type/v3/token_bucket.proto#envoy-v3-api-msg-type-v3-tokenbucket - token_bucket: - max_tokens: {{=it.platform.dapi.envoy.rateLimiter.maxTokens}} - tokens_per_fill: {{=it.platform.dapi.envoy.rateLimiter.tokensPerFill}} - fill_interval: {{=it.platform.dapi.envoy.rateLimiter.fillInterval}} - filter_enabled: - runtime_key: local_rate_limit_enabled - default_value: - numerator: {{? it.platform.dapi.envoy.rateLimiter.enabled}}100{{??}}0{{?}} - denominator: HUNDRED - filter_enforced: - runtime_key: local_rate_limit_enforced - default_value: - numerator: 100 - denominator: HUNDRED - response_headers_to_add: - - append: false - header: - key: x-local-rate-limit - value: 'true' - - name: envoy.filters.http.grpc_web - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb - - name: envoy.filters.http.cors - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - route_config: - name: local_route - virtual_hosts: - - name: dapi_services - domains: [ "*" ] - routes: - # DAPI core streaming endpoints - - match: - prefix: "/org.dash.platform.dapi.v0.Core/subscribeTo" - route: - cluster: dapi_tx_filter_stream - timeout: 660s - max_stream_duration: - grpc_timeout_header_max: 600s - # Other DAPI Core endpoints - - match: - prefix: "/org.dash.platform.dapi.v0.Core" - route: - cluster: dapi_api - timeout: {{= it.platform.dapi.envoy.http.responseTimeout }} - # DAPI waitForStateTransitionResult endpoint with bigger timeout - - match: - path: "/org.dash.platform.dapi.v0.Platform/waitForStateTransitionResult" - route: - cluster: dapi_api - timeout: 80s - # DAPI getConsensusParams endpoint - - match: - path: "/org.dash.platform.dapi.v0.Platform/getConsensusParams" - route: - cluster: dapi_api - timeout: {{= it.platform.dapi.envoy.http.responseTimeout }} - # DAPI broadcastStateTransition endpoint - - match: - path: "/org.dash.platform.dapi.v0.Platform/broadcastStateTransition" - route: - cluster: dapi_api - timeout: {{= it.platform.dapi.envoy.http.responseTimeout }} - # Drive gRPC endpoints - - match: - prefix: "/org.dash.platform.dapi.v0.Platform" - route: - cluster: drive_grpc - timeout: {{= it.platform.dapi.envoy.http.responseTimeout }} - # Static responses of unsupported api versions - # core static response - - match: - safe_regex: - google_re2: { } - regex: "\/org\\.dash\\.platform\\.dapi\\.v[1-9]+\\." - response_headers_to_add: - - header: - key: "Content-Type" - value: "application/grpc-web+proto" - - header: - key: "grpc-status" - value: "12" - - header: - key: "grpc-message" - value: "Specified service version is not supported" - direct_response: - status: 204 - # JSON RPC endpoints - - match: - path: "/" - route: - cluster: dapi_json_rpc - cors: - allow_origin_string_match: - - prefix: "*" - allow_methods: GET, PUT, DELETE, POST, OPTIONS - allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout - max_age: "1728000" - expose_headers: custom-header-1,grpc-status,grpc-message - -static_resources: - listeners: - - name: grpc_and_json_rpc - address: - socket_address: - address: 0.0.0.0 - port_value: 10000 - {{? it.platform.dapi.envoy.ssl.provider === 'self-signed'}} - listener_filters: - - name: envoy.filters.listener.tls_inspector - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector - filter_chains: - - filter_chain_match: - transport_protocol: raw_buffer - filters: *filters - - filter_chain_match: - transport_protocol: tls - filters: *filters - transport_socket: - name: envoy.transport_sockets.tls - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext - common_tls_context: - alpn_protocols: [ "h2, http/1.1" ] - tls_certificates: - - certificate_chain: - filename: "/etc/ssl/bundle.crt" - private_key: - filename: "/etc/ssl/private.key" - {{??}} - filter_chains: - filters: *filters - transport_socket: - name: envoy.transport_sockets.tls - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext - common_tls_context: - alpn_protocols: [ "h2, http/1.1" ] - tls_certificates: - - certificate_chain: - filename: "/etc/ssl/bundle.crt" - private_key: - filename: "/etc/ssl/private.key" - {{?}} - clusters: - - name: dapi_api - connect_timeout: {{= it.platform.dapi.envoy.http.connectTimeout }} - type: logical_dns - lb_policy: round_robin - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: - connection_keepalive: - interval: 30s - timeout: 5s - load_assignment: - cluster_name: dapi_api - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: dapi_api - port_value: 3005 - - name: dapi_tx_filter_stream - connect_timeout: {{= it.platform.dapi.envoy.http.connectTimeout }} - type: logical_dns - lb_policy: round_robin - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: - connection_keepalive: - interval: 30s - timeout: 5s - load_assignment: - cluster_name: dapi_tx_filter_stream - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: dapi_tx_filter_stream - port_value: 3006 - - name: dapi_json_rpc - connect_timeout: {{= it.platform.dapi.envoy.http.connectTimeout }} - type: logical_dns - lb_policy: round_robin - load_assignment: - cluster_name: dapi_json_rpc - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: dapi_api - port_value: 3004 - - name: drive_grpc - connect_timeout: {{= it.platform.dapi.envoy.http.connectTimeout }} - type: logical_dns - lb_policy: round_robin - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: - connection_keepalive: - interval: 30s - timeout: 5s - load_assignment: - cluster_name: drive_grpc - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: drive_abci - port_value: 26670 - -admin: - address: - socket_address: - address: 0.0.0.0 # For docker container only. Must be a local/private interface. - port_value: 8081 diff --git a/packages/dashmate/templates/platform/dapi/envoy/_zerossl_validation.yaml.dot b/packages/dashmate/templates/platform/gateway/_zerossl_validation.yaml.dot similarity index 100% rename from packages/dashmate/templates/platform/dapi/envoy/_zerossl_validation.yaml.dot rename to packages/dashmate/templates/platform/gateway/_zerossl_validation.yaml.dot diff --git a/packages/dashmate/templates/platform/gateway/envoy.yaml.dot b/packages/dashmate/templates/platform/gateway/envoy.yaml.dot new file mode 100644 index 0000000000..3aa31efbb9 --- /dev/null +++ b/packages/dashmate/templates/platform/gateway/envoy.yaml.dot @@ -0,0 +1,487 @@ +!ignore filters: &filters + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + normalize_path: true + merge_slashes: true + use_remote_address: true + path_with_escaped_slashes_action: UNESCAPE_AND_REDIRECT + # Settings applied both to HTTP1 and HTTP2 + common_http_protocol_options: + # A single HTTP connection timeout. + max_connection_duration: 600s + # How long to keep the connection alive when there are no streams (requests). + idle_timeout: 300s + # Request (stream) timeout. + # HTTP2 support multiple streams (requests) per connection. + # For HTTP1 it applies for single request. + # This param is overwritten in specific routes. + max_stream_duration: 15s + # Reject malformed requests with headers containing underscores. + headers_with_underscores_action: REJECT_REQUEST + # HTTP2 specific settings + http2_protocol_options: + # As a side effect this field acts as a soft limit on the number of bytes Envoy will buffer per-stream in the + # QUIC stream send and receive buffers. Once the buffer reaches this pointer, watermark callbacks will fire + # to stop the flow of data to the stream buffers. So we reduce it from 16 MiB to 64 KiB + initial_stream_window_size: 65536 # 64 KiB + # The same but for connection-level flow-control + initial_connection_window_size: 1048576 # 1 MiB + # This option sets the maximum number of concurrent streams allowed for each connection. + # It means N requests can be in flight at the same time on a single connection. + max_concurrent_streams: {{= it.platform.gateway.listeners.dapiAndDrive.http2.maxConcurrentStreams }} + # Stream idle timeout + stream_idle_timeout: 15s +{{? it.platform.gateway.log.accessLogs }} + access_log: + {{~ it.platform.gateway.log.accessLogs :log }} + {{ loggerType = {file: 'file.v3.FileAccessLog', stdout: 'stream.v3.StdoutAccessLog', stderr: 'stream.v3.StderrAccessLog'}; }} + - name: envoy.access_loggers.{{=log.type}} + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.{{=loggerType[log.type]}} + {{? log.type === 'file' }} + path: "/var/log/{{=log.path.split('/').reverse()[0]}}" + {{?? log.type === 'stream' }} + {{=log.destination}} + {{?}} + log_format: + {{? log.format === 'json' }} + json_format: + {{? log.template === null }} + timestamp: "%START_TIME%" + client: "%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%" + protocol: "%PROTOCOL%" + method: "%REQ(:METHOD)%" + uri: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + "http-status": "%RESPONSE_CODE%" + "grpc-status": "%GRPC_STATUS%" + "rx-bytes": "%BYTES_RECEIVED%" + "tx-bytes": "%BYTES_SENT%" + "response-flags": "%RESPONSE_FLAGS%" + duration: "%DURATION%" + {{??}} + {{ template = Object.entries(log.template); }} + {{~ template :entry }} + "{{=entry[0]}}": "{{=entry[1]}}" + {{~}} + {{?}} + {{??}} + text_format_source: + {{? log.template === null }} + inline_string: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%\" %RESPONSE_CODE% %GRPC_STATUS% %RESPONSE_FLAGS% R:%BYTES_RECEIVED% S:%BYTES_SENT% D:%DURATION%\n" + {{??}} + inline_string: "{{=log.template}}\n" + {{?}} + {{?}} + {{~}} +{{?}} + http_filters: + # TODO: Introduce when stable https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/adaptive_concurrency_filter.html + {{? it.platform.gateway.rateLimiter.enabled}} + - name: envoy.filters.http.ratelimit + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ratelimit.v3.RateLimit + domain: edge_proxy_per_ip + # The rate limit service timeout before the request is considered failed + timeout: 5s + # Reject a request if rate limit service is unavailable + failure_mode_deny: true + # Respond with RESOURCE_EXHAUSTED status code if request is rejected + rate_limited_as_resource_exhausted: true + rate_limit_service: + grpc_service: + envoy_grpc: + cluster_name: ratelimit_service + timeout: 0.5s + transport_api_version: V3 + {{?}} + - name: envoy.filters.http.grpc_web + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb + - name: envoy.filters.http.cors + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: local_route + virtual_hosts: + - name: http_services + domains: [ "*" ] + routes: + # DAPI core streaming endpoints + - match: + prefix: "/org.dash.platform.dapi.v0.Core/subscribeTo" + route: + cluster: dapi_core_streams + idle_timeout: 300s + # Upstream response timeout + timeout: 600s + max_stream_duration: + # Entire stream/request timeout + max_stream_duration: 600s + grpc_timeout_header_max: 600s + # Other DAPI Core endpoints + - match: + prefix: "/org.dash.platform.dapi.v0.Core" + route: + cluster: dapi_api + # Upstream response timeout + timeout: 15s + # DAPI waitForStateTransitionResult endpoint with bigger timeout + - match: + path: "/org.dash.platform.dapi.v0.Platform/waitForStateTransitionResult" + route: + cluster: dapi_api + idle_timeout: 80s + # Upstream response timeout + timeout: 80s + max_stream_duration: + # Entire stream/request timeout + max_stream_duration: 80s + grpc_timeout_header_max: 80s + # DAPI getConsensusParams endpoint + - match: + path: "/org.dash.platform.dapi.v0.Platform/getConsensusParams" + route: + cluster: dapi_api + # Upstream response timeout + timeout: 10s + # DAPI broadcastStateTransition endpoint + - match: + path: "/org.dash.platform.dapi.v0.Platform/broadcastStateTransition" + route: + cluster: dapi_api + # Upstream response timeout + timeout: 10s + # getProofs endpoint only for internal use (DAPI -> Drive) + - match: + path: "/org.dash.platform.dapi.v0.Platform/getProofs" + response_headers_to_add: + - header: + key: "Content-Type" + value: "application/grpc-web+proto" + - header: + key: "grpc-status" + value: "12" + - header: + key: "grpc-message" + value: "getProofs endpoint is only for internal use" + direct_response: + status: 204 + # Drive gRPC endpoints + - match: + prefix: "/org.dash.platform.dapi.v0.Platform" + route: + cluster: drive_grpc + # Upstream response timeout + timeout: 10s + # Static responses of unsupported api versions + # core static response + - match: + safe_regex: + regex: "\/org\\.dash\\.platform\\.dapi\\.v[1-9]+\\." + response_headers_to_add: + - header: + key: "Content-Type" + value: "application/grpc-web+proto" + - header: + key: "grpc-status" + value: "12" + - header: + key: "grpc-message" + value: "Specified service version is not supported" + direct_response: + status: 204 + # JSON RPC endpoints + - match: + path: "/" + route: + cluster: dapi_json_rpc + # Upstream response timeout + timeout: 10s + {{? it.platform.gateway.rateLimiter.enabled }} + rate_limits: + - actions: + remote_address: {} + {{?}} + typed_per_filter_config: + envoy.filters.http.cors: + "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.CorsPolicy + allow_origin_string_match: + - prefix: "*" + allow_methods: GET, PUT, DELETE, POST, OPTIONS + allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout + max_age: "1728000" + expose_headers: custom-header-1,grpc-status,grpc-message + +static_resources: + listeners: + - name: dapi_and_drive + address: + socket_address: + address: 0.0.0.0 # For docker container only. Must be a local/private interface. + port_value: 10000 + {{? it.platform.gateway.ssl.provider === 'self-signed'}} + listener_filters: + - name: envoy.filters.listener.tls_inspector + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector + per_connection_buffer_limit_bytes: 32768 # 32 KiB + filter_chains: + - filter_chain_match: + transport_protocol: raw_buffer + filters: *filters + - filter_chain_match: + transport_protocol: tls + filters: *filters + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: + alpn_protocols: [ "h2, http/1.1" ] + tls_certificates: + - certificate_chain: + filename: "/etc/ssl/bundle.crt" + private_key: + filename: "/etc/ssl/private.key" + {{??}} + filter_chains: + filters: *filters + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: + alpn_protocols: [ "h2, http/1.1" ] + tls_certificates: + - certificate_chain: + filename: "/etc/ssl/bundle.crt" + private_key: + filename: "/etc/ssl/private.key" + {{?}} + {{? it.platform.gateway.metrics.enabled }} + # Forward /stats/prometheus (which is a part of admin endpoint) + # to a separate listener with default Prometheus path /metrics + - name: prometheus_metrics + address: + socket_address: + address: "0.0.0.0" + port_value: 9090 + filter_chains: + - filters: + - name: "envoy.http_connection_manager" + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_metrics + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: [ "*" ] + routes: + - match: + prefix: "/metrics" + route: + cluster: admin + prefix_rewrite: "/stats/prometheus" + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + {{?}} + clusters: + - name: dapi_api + type: STRICT_DNS + per_connection_buffer_limit_bytes: 32768 # 32 KiB + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: + initial_stream_window_size: 65536 # 64 KiB + initial_connection_window_size: 1048576 # 1 MiB + circuit_breakers: + thresholds: + - priority: DEFAULT + # The maximum number of parallel requests + max_requests: {{= it.platform.gateway.upstreams.dapiApi.maxRequests }} + load_assignment: + cluster_name: dapi_api + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: dapi_api + port_value: 3005 + - name: dapi_core_streams + type: STRICT_DNS + per_connection_buffer_limit_bytes: 32768 # 32 KiB + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: + initial_stream_window_size: 65536 # 64 KiB + initial_connection_window_size: 1048576 # 1 MiB + circuit_breakers: + thresholds: + - priority: DEFAULT + max_requests: {{= it.platform.gateway.upstreams.dapiCoreStreams.maxRequests }} + load_assignment: + cluster_name: dapi_core_streams + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: dapi_core_streams + port_value: 3006 + - name: dapi_json_rpc + type: STRICT_DNS + per_connection_buffer_limit_bytes: 32768 # 32 KiB + circuit_breakers: + thresholds: + - priority: DEFAULT + # The maximum number of parallel connections + max_connections: {{= it.platform.gateway.upstreams.dapiJsonRpc.maxRequests }} + # The maximum number of parallel requests + max_requests: {{= it.platform.gateway.upstreams.dapiJsonRpc.maxRequests }} + load_assignment: + cluster_name: dapi_json_rpc + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: dapi_api + port_value: 3004 + - name: drive_grpc + type: STRICT_DNS + per_connection_buffer_limit_bytes: 32768 # 32 KiB + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: + initial_stream_window_size: 65536 # 64 KiB + initial_connection_window_size: 1048576 # 1 MiB + circuit_breakers: + thresholds: + - priority: DEFAULT + # The maximum number of parallel requests. + max_requests: {{= it.platform.gateway.upstreams.driveGrpc.maxRequests }} + load_assignment: + cluster_name: drive_grpc + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: drive_abci + port_value: 26670 + {{? it.platform.gateway.rateLimiter.enabled }} + - name: ratelimit_service + type: STRICT_DNS + connect_timeout: 1s + protocol_selection: USE_CONFIGURED_PROTOCOL + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: {} + load_assignment: + cluster_name: ratelimit_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: gateway_rate_limiter + port_value: 8081 + {{?}} + {{? it.platform.gateway.metrics.enabled && it.platform.gateway.admin.enabled }} + - name: admin + connect_timeout: 0.25s + type: STATIC + load_assignment: + cluster_name: admin + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: "127.0.0.1" + port_value: 9901 + {{?}} + +{{? it.platform.gateway.admin.enabled }} +admin: + address: + socket_address: + address: 0.0.0.0 # For docker container only. Must be a local/private interface. + port_value: 9901 +{{?}} + +# Dynamically adjust limits based on memory usage and number of active connections +# TODO: We can use data provided by drive, tenderdash, or dapi to configure adaptive limits based on load +# https://www.envoyproxy.io/docs/envoy/v1.30.1/api-v3/extensions/resource_monitors/injected_resource/v3/injected_resource.proto +overload_manager: + refresh_interval: 0.25s + resource_monitors: + # Monitor heap size + - name: "envoy.resource_monitors.fixed_heap" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.fixed_heap.v3.FixedHeapConfig + # Maximum heap size in bytes. If the heap size exceeds this value, Envoy will take actions to reduce memory usage. + max_heap_size_bytes: {{= it.platform.gateway.maxHeapSizeInBytes }} + # Monitor the number of active downstream connections + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: {{= it.platform.gateway.maxConnections }} + actions: + # Reduce the heap size by releasing free memory if the current heap size is 92% of the maximum heap size. + - name: "envoy.overload_actions.shrink_heap" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.92 + # Disable HTTP keepalive connections if the current heap size is 92% of the maximum heap size + # OR the number of active downstream connections is 95% of the maximum number of connections. + # Envoy will drain HTTP/2 and HTTP/3 connections using GOAWAY with a drain grace period. + # For HTTP/1, Envoy will set a drain timer to close the more idle recently used connections. + - name: "envoy.overload_actions.disable_http_keepalive" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.92 + - name: "envoy.resource_monitors.global_downstream_max_connections" + threshold: + value: 0.95 + # Stop accepting new HTTP connections in configured listeners if the number of active downstream + # connections reached the maximum. + # TODO: Use `envoy.load_shed_points.tcp_listener_accept` instead `envoy.overload_actions.stop_accepting_connections` + # when `loadshed_points` start to support `global_downstream_max_connections` monitor. + - name: "envoy.overload_actions.stop_accepting_connections" + triggers: + - name: "envoy.resource_monitors.global_downstream_max_connections" + threshold: + value: 1.0 + # Stop accepting new HTTP requests if the current heap size is 95% of the maximum heap size. + - name: "envoy.overload_actions.stop_accepting_requests" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.95 + loadshed_points: + # Stop accepting new TCP connections if the current heap size is 95% of the maximum heap size + - name: "envoy.load_shed_points.tcp_listener_accept" + triggers: + - name: "envoy.resource_monitors.fixed_heap" + threshold: + value: 0.95 diff --git a/packages/dashmate/templates/platform/gateway/rate_limiter/rate_limiter.yaml.dot b/packages/dashmate/templates/platform/gateway/rate_limiter/rate_limiter.yaml.dot new file mode 100644 index 0000000000..4e40286ada --- /dev/null +++ b/packages/dashmate/templates/platform/gateway/rate_limiter/rate_limiter.yaml.dot @@ -0,0 +1,23 @@ +domain: edge_proxy_per_ip +descriptors: + - key: remote_address + rate_limit: + unit: {{= it.platform.gateway.rateLimiter.unit }} + requests_per_unit: {{= it.platform.gateway.rateLimiter.requestsPerUnit }} + + # Blacklisted IPs + {{~ it.platform.gateway.rateLimiter.blacklist :ip }} + - key: remote_address + value: {{= ip }} + rate_limit: + unit: second + requests_per_unit: 0 + {{~}} + + # Whitelisted IPs + {{~ it.platform.gateway.rateLimiter.whitelist :ip }} + - key: remote_address + value: {{= ip }} + rate_limit: + unlimited: true + {{~}} diff --git a/packages/dashmate/templates/platform/gateway/rate_limiter/statsd_exporter.yaml.dot b/packages/dashmate/templates/platform/gateway/rate_limiter/statsd_exporter.yaml.dot new file mode 100644 index 0000000000..31f16dd6a5 --- /dev/null +++ b/packages/dashmate/templates/platform/gateway/rate_limiter/statsd_exporter.yaml.dot @@ -0,0 +1,93 @@ +mappings: # Requires statsd exporter >= v0.6.0 since it uses the "drop" action. + - match: "ratelimit.service.rate_limit.*.*.near_limit" + name: "ratelimit_service_rate_limit_near_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + - match: "ratelimit.service.rate_limit.*.*.over_limit" + name: "ratelimit_service_rate_limit_over_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + - match: "ratelimit.service.rate_limit.*.*.total_hits" + name: "ratelimit_service_rate_limit_total_hits" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + - match: "ratelimit.service.rate_limit.*.*.within_limit" + name: "ratelimit_service_rate_limit_within_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + + - match: "ratelimit.service.rate_limit.*.*.*.near_limit" + name: "ratelimit_service_rate_limit_near_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + key2: "$3" + - match: "ratelimit.service.rate_limit.*.*.*.over_limit" + name: "ratelimit_service_rate_limit_over_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + key2: "$3" + - match: "ratelimit.service.rate_limit.*.*.*.total_hits" + name: "ratelimit_service_rate_limit_total_hits" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + key2: "$3" + - match: "ratelimit.service.rate_limit.*.*.*.within_limit" + name: "ratelimit_service_rate_limit_within_limit" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + key2: "$3" + + - match: "ratelimit.service.call.should_rate_limit.*" + name: "ratelimit_service_should_rate_limit_error" + match_metric_type: counter + labels: + err_type: "$1" + + - match: "ratelimit_server.*.total_requests" + name: "ratelimit_service_total_requests" + match_metric_type: counter + labels: + grpc_method: "$1" + + - match: "ratelimit_server.*.response_time" + name: "ratelimit_service_response_time_seconds" + timer_type: histogram + labels: + grpc_method: "$1" + + - match: "ratelimit.service.config_load_success" + name: "ratelimit_service_config_load_success" + match_metric_type: counter + - match: "ratelimit.service.config_load_error" + name: "ratelimit_service_config_load_error" + match_metric_type: counter + + - match: "ratelimit.service.rate_limit.*.*.*.shadow_mode" + name: "ratelimit_service_rate_limit_shadow_mode" + timer_type: "histogram" + labels: + domain: "$1" + key1: "$2" + key2: "$3" + + # Enable below in production once you have the metrics you need + # - match: "." + # match_type: "regex" + # action: "drop" + # name: "dropped" diff --git a/packages/dashmate/test/e2e/localNetwork.spec.js b/packages/dashmate/test/e2e/localNetwork.spec.js index ca4bc23cf7..d1f8c7bb7b 100644 --- a/packages/dashmate/test/e2e/localNetwork.spec.js +++ b/packages/dashmate/test/e2e/localNetwork.spec.js @@ -52,7 +52,7 @@ describe('Local Network', function main() { localConfig.set('dashmate.helper.api.port', 40000); localConfig.set('core.p2p.port', 40001); localConfig.set('core.rpc.port', 40002); - localConfig.set('platform.dapi.envoy.http.port', 40003); + localConfig.set('platform.gateway.listeners.dapiAndDrive.port', 40003); localConfig.set('platform.drive.tenderdash.p2p.port', 40004); localConfig.set('platform.drive.tenderdash.rpc.port', 40005); localConfig.set('platform.drive.tenderdash.pprof.port', 40006); diff --git a/packages/dashmate/test/e2e/testnetEvonode.spec.js b/packages/dashmate/test/e2e/testnetEvonode.spec.js index efabb10553..f37daa0df6 100644 --- a/packages/dashmate/test/e2e/testnetEvonode.spec.js +++ b/packages/dashmate/test/e2e/testnetEvonode.spec.js @@ -93,7 +93,7 @@ describe('Testnet Evonode', function main() { config.set('dashmate.helper.api.port', 40000); config.set('core.p2p.port', 40001); config.set('core.rpc.port', 40002); - config.set('platform.dapi.envoy.http.port', 40003); + config.set('platform.gateway.listeners.dapiAndDrive.port', 40003); config.set('platform.drive.tenderdash.p2p.port', 40004); config.set('platform.drive.tenderdash.rpc.port', 40005); config.set('platform.drive.tenderdash.pprof.port', 40006); diff --git a/packages/dashmate/test/e2e/testnetFullnode.spec.js b/packages/dashmate/test/e2e/testnetFullnode.spec.js index 147b9d820d..965f6f78d6 100644 --- a/packages/dashmate/test/e2e/testnetFullnode.spec.js +++ b/packages/dashmate/test/e2e/testnetFullnode.spec.js @@ -90,7 +90,7 @@ describe('Testnet Fullnode', function main() { config.set('dashmate.helper.api.port', 40000); config.set('core.p2p.port', 40001); config.set('core.rpc.port', 40002); - config.set('platform.dapi.envoy.http.port', 40003); + config.set('platform.gateway.listeners.dapiAndDrive.port', 40003); config.set('platform.drive.tenderdash.p2p.port', 40004); config.set('platform.drive.tenderdash.rpc.port', 40005); config.set('platform.drive.tenderdash.pprof.port', 40006); diff --git a/packages/dashmate/test/unit/status/scopes/platform.spec.js b/packages/dashmate/test/unit/status/scopes/platform.spec.js index 985ff18fa3..ad723f3077 100644 --- a/packages/dashmate/test/unit/status/scopes/platform.spec.js +++ b/packages/dashmate/test/unit/status/scopes/platform.spec.js @@ -47,7 +47,7 @@ describe('getPlatformScopeFactory', () => { config = getConfigMock(this.sinon); - httpPort = config.get('platform.dapi.envoy.http.port'); + httpPort = config.get('platform.gateway.listeners.dapiAndDrive.port'); httpService = `${config.get('externalIp')}:${httpPort}`; p2pPort = config.get('platform.drive.tenderdash.p2p.port'); p2pService = `${config.get('externalIp')}:${p2pPort}`; diff --git a/packages/rs-drive-abci/.env.local b/packages/rs-drive-abci/.env.local index 2fe951bad2..942707ab20 100644 --- a/packages/rs-drive-abci/.env.local +++ b/packages/rs-drive-abci/.env.local @@ -1,8 +1,10 @@ # ABCI host and port to listen ABCI_CONSENSUS_BIND_ADDRESS="tcp://0.0.0.0:26658" -PROMETHEUS_BIND_ADDRESS="http://0.0.0.0:29090" GRPC_BIND_ADDRESS="0.0.0.0:26670" +# Metrics are disabled when empty. Must be http://0.0.0.0:29090 for example to enable +PROMETHEUS_BIND_ADDRESS= + # stderr logging for humans ABCI_LOG_STDOUT_DESTINATION=stdout ABCI_LOG_STDOUT_LEVEL=info diff --git a/packages/rs-drive-abci/.env.mainnet b/packages/rs-drive-abci/.env.mainnet index ecdae2605a..a15602a413 100644 --- a/packages/rs-drive-abci/.env.mainnet +++ b/packages/rs-drive-abci/.env.mainnet @@ -1,8 +1,10 @@ # ABCI host and port to listen ABCI_CONSENSUS_BIND_ADDRESS="tcp://0.0.0.0:26658" -PROMETHEUS_BIND_ADDRESS="http://0.0.0.0:29090" GRPC_BIND_ADDRESS="0.0.0.0:26670" +# Metrics are disabled when empty. Must be http://0.0.0.0:29090 for example to enable +PROMETHEUS_BIND_ADDRESS= + # stderr logging for humans ABCI_LOG_STDOUT_DESTINATION=stdout ABCI_LOG_STDOUT_LEVEL=info diff --git a/packages/rs-drive-abci/.env.testnet b/packages/rs-drive-abci/.env.testnet index 9cc6c3385e..e40ea7efc0 100644 --- a/packages/rs-drive-abci/.env.testnet +++ b/packages/rs-drive-abci/.env.testnet @@ -1,8 +1,10 @@ # ABCI host and port to listen ABCI_CONSENSUS_BIND_ADDRESS="tcp://0.0.0.0:26658" -PROMETHEUS_BIND_ADDRESS="http://0.0.0.0:29090" GRPC_BIND_ADDRESS="0.0.0.0:26670" +# Metrics are disabled when empty. Must be http://0.0.0.0:29090 for example to enable +PROMETHEUS_BIND_ADDRESS= + # stderr logging for humans ABCI_LOG_STDOUT_DESTINATION=stdout ABCI_LOG_STDOUT_LEVEL=info diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index db8334cc12..9540d2acf9 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -64,8 +64,8 @@ file-rotate = { version = "0.7.3" } reopen = { version = "1.0.3" } delegate = { version = "0.9.0" } regex = { version = "1.8.1" } -metrics = { version = "0.21" } -metrics-exporter-prometheus = { version = "0.12" } +metrics = { version = "0.22.3" } +metrics-exporter-prometheus = { version = "0.14.0" } url = { version = "2.3.1" } ureq = { "version" = "2.6.2" } tokio = { version = "1.36", features = [ diff --git a/packages/rs-drive-abci/src/logging/level.rs b/packages/rs-drive-abci/src/logging/level.rs index 4986c11f64..a914e45d30 100644 --- a/packages/rs-drive-abci/src/logging/level.rs +++ b/packages/rs-drive-abci/src/logging/level.rs @@ -110,7 +110,7 @@ impl TryFrom<&LogLevel> for EnvFilter { .expect("should be a valid log specification") } LogLevel::Debug => EnvFilter::try_new( - "info,tenderdash_abci=debug,drive_abci=debug,drive=debug,dpp=debug", + "info,tenderdash_abci=debug,drive_abci=debug,drive=debug,dpp=debug,dapi_grpc=debug,tonic=debug", ) .expect("should be a valid log specification"), LogLevel::Trace => EnvFilter::try_new( diff --git a/packages/rs-drive-abci/src/main.rs b/packages/rs-drive-abci/src/main.rs index 1ccaed597e..b9e07c6ea1 100644 --- a/packages/rs-drive-abci/src/main.rs +++ b/packages/rs-drive-abci/src/main.rs @@ -262,6 +262,8 @@ fn start_prometheus(config: &PlatformConfig) -> Result, Strin .filter(|s| !s.is_empty()); if let Some(addr) = prometheus_addr { + tracing::info!("Expose prometheus metrics on {}", addr); + let addr = url::Url::parse(&addr).map_err(|e| e.to_string())?; Ok(Some(Prometheus::new(addr).map_err(|e| e.to_string())?)) } else { diff --git a/packages/rs-drive-abci/src/metrics.rs b/packages/rs-drive-abci/src/metrics.rs index 5aefbc843b..be79f8d4b0 100644 --- a/packages/rs-drive-abci/src/metrics.rs +++ b/packages/rs-drive-abci/src/metrics.rs @@ -4,7 +4,8 @@ use std::{sync::Once, time::Instant}; -use metrics::{absolute_counter, describe_counter, describe_histogram, histogram, Label}; +use dapi_grpc::tonic::Code; +use metrics::{counter, describe_counter, describe_histogram, histogram, Label}; use metrics_exporter_prometheus::PrometheusBuilder; /// Default Prometheus port (29090) @@ -15,6 +16,8 @@ const COUNTER_LAST_HEIGHT: &str = "abci_last_finalized_height"; const HISTOGRAM_FINALIZED_ROUND: &str = "abci_finalized_round"; const HISTOGRAM_ABCI_REQUEST_DURATION: &str = "abci_request_duration_seconds"; const LABEL_ENDPOINT: &str = "endpoint"; +const LABEL_RESPONSE_CODE: &str = "response_code"; +const HISTOGRAM_QUERY_DURATION: &str = "abci_query_duration"; /// Error returned by metrics subsystem #[derive(thiserror::Error, Debug)] @@ -37,6 +40,7 @@ pub enum Error { pub struct HistogramTiming { key: metrics::Key, start: Instant, + skip: bool, } impl HistogramTiming { @@ -54,6 +58,7 @@ impl HistogramTiming { Self { key: metric, start: Instant::now(), + skip: false, } } @@ -61,6 +66,18 @@ impl HistogramTiming { pub fn elapsed(&self) -> std::time::Duration { self.start.elapsed() } + + /// Add label to the histrgram + pub fn add_label(&mut self, label: Label) { + self.key = self.key.with_extra_labels(vec![label]); + } + + /// Cancel timing measurement and discard the metric. + pub fn cancel(mut self) { + self.skip = true; + + drop(self); + } } impl Drop for HistogramTiming { @@ -70,11 +87,15 @@ impl Drop for HistogramTiming { /// since the start time. #[inline] fn drop(&mut self) { + if self.skip { + return; + } + let stop = self.start.elapsed(); let key = self.key.name().to_string(); let labels: Vec