From 90aba9c011f0e0ec9d4deea41c41a1ce58d5b60d Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Fri, 24 Mar 2023 17:10:12 -0700 Subject: [PATCH] V1 (#383) --- .github/workflows/publish-ci-docker-image.yml | 6 +- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 388 ++++-------------- images/instacart.webp | Bin 0 -> 3448 bytes images/postgresml.webp | Bin 0 -> 4862 bytes 6 files changed, 79 insertions(+), 319 deletions(-) create mode 100644 images/instacart.webp create mode 100644 images/postgresml.webp diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml index 3cae8e7e..5df4fd48 100644 --- a/.github/workflows/publish-ci-docker-image.yml +++ b/.github/workflows/publish-ci-docker-image.yml @@ -15,6 +15,6 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build CI Docker image run: | - docker build . -f Dockerfile.ci --tag ghcr.io/levkk/pgcat-ci:latest - docker run ghcr.io/levkk/pgcat-ci:latest - docker push ghcr.io/levkk/pgcat-ci:latest + docker build . -f Dockerfile.ci --tag ghcr.io/postgresml/pgcat-ci:latest + docker run ghcr.io/postgresml/pgcat-ci:latest + docker push ghcr.io/postgresml/pgcat-ci:latest diff --git a/Cargo.lock b/Cargo.lock index 9856f6e0..9124996f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -716,7 +716,7 @@ dependencies = [ [[package]] name = "pgcat" -version = "0.6.0-alpha1" +version = "1.0.0" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 4be3c767..e5b1de66 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgcat" -version = "0.6.0-alpha1" +version = "1.0.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/README.md b/README.md index dc42d0d5..28433def 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,47 @@ -##### PgCat: PostgreSQL at petabyte scale +## PgCat: Nextgen PostgreSQL Pooler [![CircleCI](https://circleci.com/gh/postgresml/pgcat/tree/main.svg?style=svg)](https://circleci.com/gh/postgresml/pgcat/tree/main) Join our Discord! -PostgreSQL pooler (like PgBouncer) with sharding, load balancing and failover support. - -**Beta**: looking for beta testers, see [#35](https://github.com/levkk/pgcat/issues/35). +PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load balancing, failover and mirroring. ## Features -| **Feature** | **Status** | **Comments** | -|--------------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| -| Transaction pooling | :white_check_mark: | Identical to PgBouncer. | -| Session pooling | :white_check_mark: | Identical to PgBouncer. | -| `COPY` support | :white_check_mark: | Both `COPY TO` and `COPY FROM` are supported. | -| Query cancellation | :white_check_mark: | Supported both in transaction and session pooling modes. | -| Load balancing of read queries | :white_check_mark: | Using random between replicas. Primary is included when `primary_reads_enabled` is enabled (default). | -| Sharding | :white_check_mark: | Transactions are sharded using `SET SHARD TO` and `SET SHARDING KEY TO` syntax extensions; see examples below. | -| Failover | :white_check_mark: | Replicas are tested with a health check. If a health check fails, remaining replicas are attempted; see below for algorithm description and examples. | -| Statistics | :white_check_mark: | Statistics available in the admin database (`pgcat` and `pgbouncer`) with `SHOW STATS`, `SHOW POOLS` and others. | -| Live configuration reloading | :white_check_mark: | Reload supported settings with a `SIGHUP` to the process, e.g. `kill -s SIGHUP $(pgrep pgcat)` or `RELOAD` query issued to the admin database. | -| Client authentication | :white_check_mark: :wrench: | MD5 password authentication is supported, SCRAM is on the roadmap; one user is used to connect to Postgres with both SCRAM and MD5 supported. | -| Admin database | :white_check_mark: | The admin database, similar to PgBouncer's, allows to query for statistics and reload the configuration. | + +| **Feature** | **Status** | **Comments** | +|-------------|------------|--------------| +| Transaction pooling | **Stable** | Identical to PgBouncer with notable improvements for handling bad clients and abandoned transactions. | +| Session pooling | **Stable** | Identical to PgBouncer. | +| Multi-threaded runtime | **Stable** | Using Tokio asynchronous runtime, the pooler takes advantage of multicore machines. | +| Load balancing of read queries | **Stable** | Queries are automatically load balanced between replicas and the primary. | +| Failover | **Stable** | Queries are automatically rerouted around broken replicas, validated by regular health checks. | +| Admin database statistics | **Stable** | Pooler statistics and administration via the `pgbouncer` and `pgcat` databases. | +| Prometheus statistics | **Stable** | Statistics are reported via a HTTP endpoint for Prometheus. | +| Client TLS | **Stable** | Clients can connect to the pooler using TLS/SSL. | +| Client/Server authentication | **Stable** | Clients can connect using MD5 authentication, supported by `libpq` and all Postgres client drivers. PgCat can connect to Postgres using MD5 and SCRAM-SHA-256. | +| Live configuration reloading | **Stable** | Identical to PgBouncer; all settings can be reloaded dynamically (except `host` and `port`). | +| Sharding using extended SQL syntax | **Experimental** | Clients can dynamically configure the pooler to route queries to specific shards. | +| Sharding using comments parsing/Regex | **Experimental** | Clients can include shard information (sharding key, shard ID) in the query comments. | +| Automatic sharding | **Experimental** | PgCat can parse queries detect sharding keys automatically, and route queries to the route shard. | +| Mirroring | **Experimental** | Mirror queries between multiple databases in order to test servers with realistic production traffic. | + + +## Status + +PgCat is stable and used in production to serve hundreds of thousands of queries per second. Some features remain experimental and are being actively developed. They are optional and can be enabled through configuration. + +| | | +|-|-| +||| +| [Instacart](https://tech.instacart.com/adopting-pgcat-a-nextgen-postgres-proxy-3cf284e68c2f) | [PostgresML](https://postgresml.org/blog/scaling-postgresml-to-one-million-requests-per-second) | ## Deployment See `Dockerfile` for example deployment using Docker. The pooler is configured to spawn 4 workers so 4 CPUs are recommended for optimal performance. That setting can be adjusted to spawn as many (or as little) workers as needed. +A Docker image is available from `docker pull ghcr.io/postgresml/pgcat:latest`. See our [Github packages repository](https://github.com/postgresml/pgcat/pkgs/container/pgcat). + For quick local example, use the Docker Compose environment provided: ```bash @@ -39,9 +53,13 @@ PGPASSWORD=postgres psql -h 127.0.0.1 -p 6432 -U postgres -c 'SELECT 1' ### Config -See [Configurations page](https://github.com/levkk/pgcat/blob/main/CONFIG.md) +See **[Configuration](https://github.com/levkk/pgcat/blob/main/CONFIG.md)**. -## Local development +## Contributing + +The project is being actively developed and looking for additional contributors and production deployments. + +### Local development 1. Install Rust (latest stable will work great). 2. `cargo build --release` (to get better benchmarks). @@ -51,7 +69,7 @@ See [Configurations page](https://github.com/levkk/pgcat/blob/main/CONFIG.md) ### Tests -Quickest way to test your changes is to use pgbench: +When making substantial modifications to the protocol implementation, make sure to test them with pgbench: ``` pgbench -i -h 127.0.0.1 -p 6432 && \ @@ -61,36 +79,26 @@ pgbench -t 1000 -p 6432 -h 127.0.0.1 --protocol extended See [sharding README](./tests/sharding/README.md) for sharding logic testing. -Run `cargo test` to run Rust tests. +Additionally, all features are tested with Ruby, Python, and Rust tests unit and integration tests. + +Run `cargo test` to run Rust unit tests. + +Run the following commands to run Ruby and Python integration tests: -Run the following commands to run Integration tests locally. ``` cd tests/docker/ docker compose up --exit-code-from main # This will also produce coverage report under ./cov/ ``` -| **Feature** | **Tested in CI** | **Tested manually** | **Comments** | -|-----------------------|--------------------|---------------------|--------------------------------------------------------------------------------------------------------------------------| -| Transaction pooling | :white_check_mark: | :white_check_mark: | Used by default for all tests. | -| Session pooling | :white_check_mark: | :white_check_mark: | Tested by running pgbench with `--protocol prepared` which only works in session mode. | -| `COPY` | :white_check_mark: | :white_check_mark: | `pgbench -i` uses `COPY`. `COPY FROM` is tested as well. | -| Query cancellation | :white_check_mark: | :white_check_mark: | `psql -c 'SELECT pg_sleep(1000);'` and press `Ctrl-C`. | -| Load balancing | :white_check_mark: | :white_check_mark: | We could test this by emitting statistics for each replica and compare them. | -| Failover | :white_check_mark: | :white_check_mark: | Misconfigure a replica in `pgcat.toml` and watch it forward queries to spares. CI testing is using Toxiproxy. | -| Sharding | :white_check_mark: | :white_check_mark: | See `tests/sharding` and `tests/ruby` for an Rails/ActiveRecord example. | -| Statistics | :white_check_mark: | :white_check_mark: | Query the admin database with `psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW STATS'`. | -| Live config reloading | :white_check_mark: | :white_check_mark: | Run `kill -s SIGHUP $(pgrep pgcat)` and watch the config reload. | - -### Dev +### Docker-based local development -Also, you can open a 'dev' environment where you can debug tests easier by running the following command: +You can open a Docker development environment where you can debug tests easier. Run the following command to spin it up: ``` ./dev/script/console ``` -This will open a terminal in an environment similar to that used in tests. In there you can compile, run tests, do some debugging with the test environment, etc. Objects -compiled inside the contaner (and bundled gems) will be placed in `dev/cache` so they don't interfere with what you have in your host. +This will open a terminal in an environment similar to that used in tests. In there, you can compile the pooler, run tests, do some debugging with the test environment, etc. Objects compiled inside the contaner (and bundled gems) will be placed in `dev/cache` so they don't interfere with what you have on your machine. ## Usage @@ -105,11 +113,9 @@ In transaction mode, a client talks to one server for the duration of a single t This mode is enabled by default. ### Load balancing of read queries -All queries are load balanced against the configured servers using the random algorithm. The most straight forward configuration example would be to put this pooler in front of several replicas and let it load balance all queries. - -If the configuration includes a primary and replicas, the queries can be separated with the built-in query parser. The query parser will interpret the query and route all `SELECT` queries to a replica, while all other queries including explicit transactions will be routed to the primary. +All queries are load balanced against the configured servers using either the random or least open connections algorithms. The most straightforward configuration example would be to put this pooler in front of several replicas and let it load balance all queries. -The query parser is disabled by default. +If the configuration includes a primary and replicas, the queries can be separated with the built-in query parser. The query parser, implemented with the `sqlparser` crate, will interpret the query and route all `SELECT` queries to a replica, while all other queries including explicit transactions will be routed to the primary. #### Query parser The query parser will do its best to determine where the query should go, but sometimes that's not possible. In that case, the client can select which server it wants using this custom SQL syntax: @@ -136,38 +142,14 @@ The setting will persist until it's changed again or the client disconnects. By default, all queries are routed to the first available server; `default_role` setting controls this behavior. ### Failover -All servers are checked with a `SELECT 1` query before being given to a client. If the server is not reachable, it will be banned and cannot serve any more transactions for the duration of the ban. The queries are routed to the remaining servers. If all servers become banned, the ban list is cleared: this is a safety precaution against false positives. The primary can never be banned. +All servers are checked with a `;` (very fast) query before being given to a client. Additionally, the server health is monitored with every client query that it processes. If the server is not reachable, it will be banned and cannot serve any more transactions for the duration of the ban. The queries are routed to the remaining servers. If all servers become banned, the ban list is cleared: this is a safety precaution against false positives. The primary can never be banned. The ban time can be changed with `ban_time`. The default is 60 seconds. -Failover behavior can get pretty interesting (read complex) when multiple configurations and factors are involved. The table below will try to explain what PgCat does in each scenario: - -| **Query** | **`SET SERVER ROLE TO`** | **`query_parser_enabled`** | **`primary_reads_enabled`** | **Target state** | **Outcome** | -|---------------------------|--------------------------|----------------------------|-----------------------------|------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Read query, i.e. `SELECT` | unset (any) | false | false | up | Query is routed to the first instance in the random loop. | -| Read query | unset (any) | true | false | up | Query is routed to the first replica instance in the random loop. | -| Read query | unset (any) | true | true | up | Query is routed to the first instance in the random loop. | -| Read query | replica | false | false | up | Query is routed to the first replica instance in the random loop. | -| Read query | primary | false | false | up | Query is routed to the primary. | -| Read query | unset (any) | false | false | down | First instance is banned for reads. Next target in the random loop is attempted. | -| Read query | unset (any) | true | false | down | First replica instance is banned. Next replica instance is attempted in the random loop. | -| Read query | unset (any) | true | true | down | First instance (even if primary) is banned for reads. Next instance is attempted in the random loop. | -| Read query | replica | false | false | down | First replica instance is banned. Next replica instance is attempted in the random loop. | -| Read query | primary | false | false | down | The query is attempted against the primary and fails. The client receives an error. | -| | | | | | | -| Write query e.g. `INSERT` | unset (any) | false | false | up | The query is attempted against the first available instance in the random loop. If the instance is a replica, the query fails and the client receives an error. | -| Write query | unset (any) | true | false | up | The query is routed to the primary. | -| Write query | unset (any) | true | true | up | The query is routed to the primary. | -| Write query | primary | false | false | up | The query is routed to the primary. | -| Write query | replica | false | false | up | The query is routed to the replica and fails. The client receives an error. | -| Write query | unset (any) | true | false | down | The query is routed to the primary and fails. The client receives an error. | -| Write query | unset (any) | true | true | down | The query is routed to the primary and fails. The client receives an error. | -| Write query | primary | false | false | down | The query is routed to the primary and fails. The client receives an error. | -| | | | | | | - ### Sharding We use the `PARTITION BY HASH` hashing function, the same as used by Postgres for declarative partitioning. This allows to shard the database using Postgres partitions and place the partitions on different servers (shards). Both read and write queries can be routed to the shards using this pooler. +#### Extended syntax To route queries to a particular shard, we use this custom SQL syntax: ```sql @@ -182,7 +164,8 @@ The active shard will last until it's changed again or the client disconnects. B For hash function implementation, see `src/sharding.rs` and `tests/sharding/partition_hash_test_setup.sql`. -#### ActiveRecord/Rails + +##### ActiveRecord/Rails ```ruby class User < ActiveRecord::Base @@ -210,7 +193,7 @@ User.connection.execute "SET SERVER ROLE TO 'auto'" User.find_by_email("test@example.com") ``` -#### Raw SQL +##### Raw SQL ```sql -- Grab a bunch of users from shard 1 @@ -230,268 +213,45 @@ SET SERVER ROLE TO 'auto'; -- let the query router figure out where the query sh SELECT * FROM users WHERE email = 'test@example.com'; -- shard setting lasts until set again; we are reading from the primary ``` -### Statistics reporting - -The stats are very similar to what Pgbouncer reports and the names are kept to be comparable. They are accessible by querying the admin database `pgcat`, and `pgbouncer` for compatibility. +#### With comments +Issuing queries to the pooler can cause additional latency. To reduce its impact, it's possible to include sharding information inside SQL comments sent via the query. This is reasonably easy to implement with ORMs like [ActiveRecord](https://api.rubyonrails.org/classes/ActiveRecord/QueryMethods.html#method-i-annotate) and [SQLAlchemy](https://docs.sqlalchemy.org/en/20/core/events.html#sql-execution-and-connection-events). ``` -psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' -``` - -### Live configuration reloading - -The config can be reloaded by sending a `kill -s SIGHUP` to the process or by querying `RELOAD` to the admin database. Not all settings are currently supported by live reload: - -| **Config** | **Requires restart** | -|-------------------------|----------------------| -| `host` | yes | -| `port` | yes | -| `pool_mode` | no | -| `connect_timeout` | yes | -| `healthcheck_timeout` | no | -| `shutdown_timeout` | no | -| `healthcheck_delay` | no | -| `ban_time` | no | -| `user` | yes | -| `shards` | yes | -| `default_role` | no | -| `primary_reads_enabled` | no | -| `query_parser_enabled` | no | - - -## Benchmarks +/* shard_id: 5 */ SELECT * FROM foo WHERE id = 1234; -You can setup PgBench locally through PgCat: - -``` -pgbench -h 127.0.0.1 -p 6432 -i -``` - -Coincidenly, this uses `COPY` so you can test if that works. Additionally, we'll be running the following PgBench configurations: - -1. 16 clients, 2 threads -2. 32 clients, 2 threads -3. 64 clients, 2 threads -4. 128 clients, 2 threads - -All queries will be `SELECT` only (`-S`) just so disks don't get in the way, since the dataset will be effectively all in RAM. - -My setup: - -- 8 cores, 16 hyperthreaded (AMD Ryzen 5800X) -- 32GB RAM (doesn't matter for this benchmark, except to prove that Postgres will fit the whole dataset into RAM) - -### PgBouncer - -#### Config - -```ini -[databases] -shard0 = host=localhost port=5432 user=sharding_user password=sharding_user - -[pgbouncer] -pool_mode = transaction -max_client_conn = 1000 +/* sharding_key: 1234 */ SELECT * FROM foo WHERE id = 1234; ``` -Everything else stays default. +#### Automatic query parsing +PgCat can use the `sqlparser` crate to parse SQL queries and extract the sharding key. This is configurable with the `automatic_sharding_key` setting. This feature is still experimental, but it's the ideal implementation for sharding, requiring no client modifications. -#### Runs +### Statistics reporting +The stats are very similar to what PgBouncer reports and the names are kept to be comparable. They are accessible by querying the admin database `pgcat`, and `pgbouncer` for compatibility. ``` -$ pgbench -t 1000 -c 16 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 16 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 16000/16000 -latency average = 0.155 ms -tps = 103417.377469 (including connections establishing) -tps = 103510.639935 (excluding connections establishing) - - -$ pgbench -t 1000 -c 32 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 32 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 32000/32000 -latency average = 0.290 ms -tps = 110325.939785 (including connections establishing) -tps = 110386.513435 (excluding connections establishing) - - -$ pgbench -t 1000 -c 64 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 64 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 64000/64000 -latency average = 0.692 ms -tps = 92470.427412 (including connections establishing) -tps = 92618.389350 (excluding connections establishing) - -$ pgbench -t 1000 -c 128 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended shard0 - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 128 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 128000/128000 -latency average = 1.406 ms -tps = 91013.429985 (including connections establishing) -tps = 91067.583928 (excluding connections establishing) +psql -h 127.0.0.1 -p 6432 -d pgbouncer -c 'SHOW DATABASES' ``` -### PgCat +Additionally, Prometheus statistics are available at `/metrics` via HTTP. -#### Config +### Live configuration reloading -The only thing that matters here is the number of workers in the Tokio pool. Make sure to set it to < than the number of your CPU cores. -Also account for hyper-threading, so if you have that, take the number you got above and divide it by two, that way only "real" cores serving -requests. +The config can be reloaded by sending a `kill -s SIGHUP` to the process or by querying `RELOAD` to the admin database. All settings except the `host` and `port` can be reloaded without restarting the pooler, including sharding and replicas configurations. -My setup is 16 threads, 8 cores (`htop` shows as 16 CPUs), so I set the `max_workers` in Tokio to 4. Too many, and it starts conflicting with PgBench -which is also running on the same system. +### Mirroring -#### Runs +Mirroring allows to route queries to multiple databases at the same time. This is useful for prewarning replicas before placing them into the active configuration, or for testing different versions of Postgres with live traffic. +## License -``` -$ pgbench -t 1000 -c 16 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 16 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 16000/16000 -latency average = 0.164 ms -tps = 97705.088232 (including connections establishing) -tps = 97872.216045 (excluding connections establishing) - - -$ pgbench -t 1000 -c 32 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 32 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 32000/32000 -latency average = 0.288 ms -tps = 111300.488119 (including connections establishing) -tps = 111413.107800 (excluding connections establishing) - - -$ pgbench -t 1000 -c 64 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 64 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 64000/64000 -latency average = 0.556 ms -tps = 115190.496139 (including connections establishing) -tps = 115247.521295 (excluding connections establishing) - -$ pgbench -t 1000 -c 128 -j 2 -p 6432 -h 127.0.0.1 -S --protocol extended - -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 128 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 128000/128000 -latency average = 1.135 ms -tps = 112770.562239 (including connections establishing) -tps = 112796.502381 (excluding connections establishing) -``` +PgCat is free and open source, released under the MIT license. -### Direct Postgres +## Contributors -Always good to have a base line. +Many thanks to our amazing contributors! -#### Runs + + + -``` -$ pgbench -t 1000 -c 16 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 16 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 16000/16000 -latency average = 0.115 ms -tps = 139443.955722 (including connections establishing) -tps = 142314.859075 (excluding connections establishing) - -$ pgbench -t 1000 -c 32 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 32 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 32000/32000 -latency average = 0.212 ms -tps = 150644.840891 (including connections establishing) -tps = 152218.499430 (excluding connections establishing) - -$ pgbench -t 1000 -c 64 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 64 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 64000/64000 -latency average = 0.420 ms -tps = 152517.663404 (including connections establishing) -tps = 153319.188482 (excluding connections establishing) - -$ pgbench -t 1000 -c 128 -j 2 -p 5432 -h 127.0.0.1 -S --protocol extended shard0 -Password: -starting vacuum...end. -transaction type: -scaling factor: 1 -query mode: extended -number of clients: 128 -number of threads: 2 -number of transactions per client: 1000 -number of transactions actually processed: 128000/128000 -latency average = 0.854 ms -tps = 149818.594087 (including connections establishing) -tps = 150200.603049 (excluding connections establishing) -``` diff --git a/images/instacart.webp b/images/instacart.webp new file mode 100644 index 0000000000000000000000000000000000000000..a3aa1fca560ae0f6b6af8ec1aea6b25d02ebc223 GIT binary patch literal 3448 zcmaJ@c{o*DA6^_o8A1qUI5JckMCN18Sm=;h9F8foPGpLM%8+Cx2^mY79YeZxj3q>v`5*?|y&pyY_GW_S*YdCi*%$zN`RPXltU)(F&GK z001Q1cL0b3K>zY(^mAJH7SKNTbo9DScM$+DZ$F%gt|rpb${NYI01$u|@W6so4vx2d zHPC2-1NHxITVDXpL1gg!!Ce1a^N%d6lk;sy0BDe~Lfz39=LchZ7|R9t`5xdWFy?Z+ z>EHz8QW&4Z!3n~6^nl0yiI)$s>o5FkQCv7vlgsehxL_XX`ZtXI8+N>j^MW-putwC$ z%Nw4b*78s6bb#+3U@uR9IJaNzV1+Es-e%X~Q55d{Ko{r(Lx6_MiUt0_4R``Sa2_7L zVdge4f#aJ0;6MJ$kA_`X*yRQsVTTs*0bao2mmeIw16TvMziNBi<&yL-iiSlK0QxBi z`Y8qgV+sJ2L2L&Sln8LL(=m8+}DFEWG@Z2~4 zVcmyd!+-1ce>MN9AG~vsui&j)sR*)ZB*@?qP?h41(>f`f$6Q zXStuvq+0JGZG}3*PRz2L5(uEy^!0A7x#BZa!qtq39eAUCr`4`K;qM5t#b&}TenzEG z&6YB>3-G4U`6d13)SrWZsF7+P@r_|dRI^(<-Hb1J3a8INYLuF7wk)AiwF&547#IF5 zBt(wx@laQr#|%O<(5?N#T@&hch=+Vl8IuM*gnFpD-$cmr8_-tnvuWNb@0`zSm9w<_ zrPUX>)3sZwI~vTbcogCtlX#fAgix-b7tt;JXBaCg#^?C%b(cz|T~DajBVN&$sQ-?0 z-O;J2z$db6*kOA=#LO+|iL)0;FXcqW+>DN-9M4b2?EEJ9QtS$Lq=J_K98JrA$`~By z=#!XZ(ku=#R0uJzSbl6WW``2mr?FG<*jZKaL8M1VXGewKKeG48F8XQyhn=>U=#}i; z0a4-pagh{_%EZFmkYszN_XF<-r8s>uSMA=2%7n_1YeiTUwdV;rnV~8?i!pHi%IfsL z@^|i_>#$-Lb~^VURUJ^FFY}*^*s5~`rt}V9a3z|_dfsc*#7Z?0)M7ou*V^jT1Aezb zlKN6^z#lwHf@@Q6eEUnByoCQN4BxD3$=AoLT{3suYmZdlvEhnYA90iISIg88AO_26@!URB(|kUK8!!s$b!{Ax_yFR6xFJ;Y$t^UK$M3P7#g zXT={bew`ni+{v+kb68fntkkBIQ$B`_#S$cLpkl-b*bGg=cY-hp#32>%mv}P zX6kAf+f3H<$q|wqS4L7r#8S+?Wy+l0rRLVU)$UabYS$wngx*K>!TN_suRIjLWwS>s z{{kb7FmEsJU@X`2xAru-ui1{i$CIP{f(qHw9}&2vY~7bTZPbmo{_yTFqw{HDnnYiO z23bwmxmoopZ+)!bWa(7y4tlo5yRwh`d7)7+GdO+4jhOPi#sX!H9O0N{Rw++Q5*B}> z9jN-8R&kDx_|V~Wc2UL@^{vrcvo>b}X$yhMxG*2WntJHjQRZDJ_s{QT4p-=`6y z?;GbyYyvgYZPA4|ul=1$)6VmqXu6MdCYVH~ zv#LGJ4?Ro#Jc7eZouhd2P+Gn2CqrHZE;e+3w(NYdfxp`C`}$kVVu9A2QXkK?^Xu0f zpbSGf^OcgTDHyRTS6`p+M?INJ*Br?jb+Q)oO!4cPg8akuGYKn7T_b52#^i;_lkJi8 zYDH3R`^;6GuZ0?-7s}v2i2>)%Fb46Fyj>&{KyYu=l{J4_gRb5($5Gi2ec=Y5550U1-8?!`>GB%UHIn|K$dG;lt&!~2s-G%qx=_t= z^Y{A4Sc{&JdNb=6N6ZO{lBsuY_3aN>v^};y#LP{SIWd$h!JKm;xVf(Ljc}T|Ypl0z zYpiDEJKEui6xFHMy>y1NoHOq14Yd^25*^3pFmc{kt0D%`HZE>=O`N^?S#OoLkF4(( zb*nxHJmz=!ZU4u%OE7mj(UKZWxE^k7IwWZR$tRn4W*4ZBZQM67A3x?fd`QQGkf2lXH7x3kS3a#M?p;dsW0u~R z&5d)jPhP*o4Er;w5+;HSAtjqW6m;4_YWyzr?X{fX_MFk2-BNI{H@@|n_# zQJP+<+TMbstx&qUo5JbZQBfuAC5{d6NJJ0L?+%{mK|YjbK*-*sem&~+D@#_xYD z5Q5nRuxmtRXC?O3f*&2Wi{5vRXi1Ldt4?WM3GU$K%M9WQTcKxZ<77Y7yX-i;OnRd9 zNr1%HD)0Do>O`O0$#{bKHU3<&jok1XsDc%$YuHM?hsPa=YR1@NbWTcFR3?BHYA@(m zZF^asPIcp{QY{jEdYJ03`y|OB3`^^=d=ER{ajHU5H`E;Hv znR?po_qfk#{}Ol4x32!-j#%5nRi092Y(qLDRJSAAejwgz%(5*;*+mYKlQZI37dk0v zHqs;}?MX&^dD6bov-ScmCT{m&s+^96*4FA+wXJ*>^tfPrzLq@?Db`wSSt`-+D#X5f zQr6;Pgy6i*v*FJi`vUFDIZ*r3UQOBz!D8_&Ca;eAO548W)9N1{t3r+ZHUv z#<5(ToK#3V^8hsQ#f49IxGbU}^e^-Ccp6xiBoms6xV_Bt`!ul;1#-+iC zEqn&aNL{-41Dj1S%`u$BP2&aGi4(p0Au23hH&rGevZ)Yjjx8;tXc4*Pan2hbe`Iyj zfa)Z_duQ1XFWe)ijH^Q>Y5Ps*JGNfG&kk9mp8vw5%iLu?6LufVM2MD6-5 z61#jXaB;$hW_#F!{m#Y$qe1Lb!YJ{hcHAbH zfl;!~QD{2pc2{xFj!cyJX(C5VoN(>nNXcQ5Ci&jqmhW$MI@eY-G7K;$SXDg8e|t}! zIey%>Nr8Gd!27Aw5nmc^cD*tzj&bH6HG|>j`PflSB(ug)X`>6WkJsT0hw$(1gz?KFy4=84c(g)y6Uk1BQ~b`WgLi z>}Z5cBTQ^~P^v*MLg?l2qBEO%I?L6j7P!DS~bTI=5TbKh&P=UG4YzP7%`qesUy0AQ@Dq-&%rX+j17fDo=* z0YE$eXecY|7USb?0sLZD8;BPnE(C#kd+Mtxv74AaWhdDH2ml7)7EZupW#i?psH>}a zrT*{jcmlw?ihLKiTI;`S{*|S)wezw806cb_Lczw}(;J5^aai2X+x-e><1np_qm?ZV z*WfU}CvJl{Jaxre|BZL9u)|;YucT-_4fU0AxzXZyc8CAK*8hQR96cd8jVMk7vV}l# z>*JgJjcu=R&=rQb`r!8VSG&rP!VU^Hz|A0BX9m;&4L}Rf1=s;=zz1*wTmf%D05?N% zj2EDfi!1$$pYbohF3x3*b2$MvIEM=020(DfbL9uF&H$&uwSQ{sWiRsJFA9%B2>^%| zu-GFm03b;N08BC#dtQRYV&DKkFb4pg(0}}H#Q-4r8^^z;T4@deKu#_I za691Ee)cbUU&9&xuWtXZ=6~y_sHv;2j&tD^f5bF8HJeXJZwwd2?v_Ck-t2zXfDj^A z#v34NxbgAsRD)DuvtKYw(xDc4oWn$YQ=wTnyA9$fw|0C%wl5gfEHHPic(c4Mzkv`q zl7M)!gi>N2G7E358~t$6V!r35laojgC%>ImCAURGa?+k_PRgePI!ic&E z(gKhr@a4kO{D`of>i3v0_mbOvzy98MM()+In(If5W~0+PmL#F(X|;}YCrr9zgKRD) zwEVF`y%fcVA038gBz}E6Ii*Oa2n~}^B4DHx0uY_%9iC$Zw^;S%S+@0ASQjV^h)Z;ldVh{$&-*P-rpF zOR2yUK||!5Lj!^OdGOK$YBTr4{f0O>TCR-X-Kx?SB;%A9^RPhy9ztF_<{Sd%;M{mO z@LiOzihHNDDx&-|y*m=ICYfJa0tT{GDQpM%s);(8n{~vfwyPR-wwgvl&i^#;z1iZc zlC(N-tG?8?BPL$Gp6AUb72>P6+yTbC$$h+Ce4ybMf{a#m?Se~HO;?foyhpx}6)T!v z3T}38u4<>IUSrwc_tgF@t@Wb5ysd+U`n67>Nk*hyE_a|`^Q4UFpB z8+h*A%g;ty_etbh#Ml|@nZs!F8@j*9T6hKI=kYOtLMy+W)I|FDIqyvZ0yK(U;Me$E z1d>`JipJR#@SBbGL<9<-I^urvm?xEuDx&zUZ;`~gXY47NTlB08913jy+%*zRsWWPZ zmaXqo8STBR+_qAhf0B6S++xeJ^Fq_Zkrci$HHtJ%LlWg)j()80r+H6xI|`lsP;rKX z7#dCiRwuj zfPG@1nV~F_VCoRE(6Fy3-L!h8B}|BON791yZ$UJb&XBq$9YxA@_73NM9qb-nsMuRj zmyQRs@kG?PKs8~v8v&7zMW684R#>QZte2enS*c}nYr{*jtjfAz&=wKA8A(Z;S6mdA z+crq{kh4TDwa%u0Urs#0R86r+zOGJ2MIi1DMJSqk(2v7zG8dmk&wn`^8E&ej`bz^O z%?UZIQ2!#!A9ELmjYkKc4oN(3-xy~Ep*KwCviT!+@Az7Urv!HmWba1V&eSP^R@;Zt zRFhQVUh&M@Oc6JUSVe&qrcG^wNNXm4XwU~Ijn(&`;Q_AU9HcTQJqG<3oY`)U$^1oK zAAS9WjK}A8tlT!3zopXcSOd??J(c|&r>WOao;_m1{J;}Lg%Hm}Ive?Fi~HG2r05FI z&Iivg6q4H%4BTo3ZKFZ&ehU_r+eO`Wt7J|`{GupM+u>I??WRe8P7m4+0s01^)YA@7 zN*Zq~u22zMn=bAeh2yC3-d_U5p*HYviHLNSEJv!({35COu}7E%=u62MCNv(p|~GX8ecMtrg{L!ueS-?`RP* zJmPoY#5Q#giVwfPV(rF5-53Oh>0||xAB0Nn#@=nJn(7i(IF(T&7PrFYP`HC;%tVJk zzF@OEa5K4HCa{jkDomGFp3HVy)6|HRerg zG4#F@LZ+L$^PGcPFKL-U%y=*N98)$@aoJMGPtGCZJD$nhNo__<6*Kx%Gg7SMRvE}a zapXFH+ExpnyrX1<_-32B0A>Fuq2zOGF!eOXF~7nG$vG~V)^x4x*S^+?Rz6#tE{@De>?AjwO33)L)Y|)UTOP-SyQWLLV}7#sZBTZ#dCLylOR)( z=L5q&LOWOwS!Wv=d4bmNOSaT~!D_=x7AC%2{fiUX9cac?)sDK%Wi;W#WoZdvWsSmx zj8MgQUd%f#rj)9)k6}4Vld9Lz8!2^cgA6s60MpSkWtv0CV-fsy=GYMRJ(5(tiNp`J z;cp`eh|r)FZG9{tVYUkeoeYNKA$BJ5!Cj1DY2cN9bGPWI5C;c+iNVeB92>&uo=mQU zufSN*HMEFw*Y2YMo__b3IB*Ut=t2z7;7Gv2Xp4t99CMG5qT#fj!}QX&3ogr|?oSfr zY3t!tM
bo(a<=FaqB>4pK zKv3S7J?X7UcU)b(`ZEXmhxGILI&$aTK@M&-(KekPH7iam{pdts%62OZ#5ROj52$## z8o-=TA>*|T=lCe)``A;Sz)B}321N`pvuu)D` zjh$rSDN+8UbP`CE)t29+L`h^J8Nfe-aWoifFsFpr_nb#O)$??FrSA7|BLCMP#R0RB zKujFQAbs-y{`F#s@=M+KEZF7gIhDa;;NGhzI*#V?sn{aR)9-50>1pvczC&#rF`{eR zv7fLVvfE9>>4SKZ*^qgwbq&6yx<(wVX$emTD?vi)&jGLRYd1qEPhD(!ghq|l#hZjU~gl* z&XZ6i0c;Z38u?AwhfC~57Z#2Q{ljxMxr28xc?PRlJae;>pIx-1nu38B4`F8?KLwG8 z48Yd0vip2zC~WH#!6@!}>##Y3QXCLMuw(*7TcR$kB%Pmsa6uEr)!Rwfc|VKn>8B(3 zDw3nLornx4hc)Hk0nvh{nfLtD*f3bMnQQ9CqhL)Sg6|~{CVU}hUU8|?; z6V**I0@A#SV*=&!CzQK?z1oV#1f1si?*7XC42i5!v! z!P`#O*_-A2?eOkd<#EV|n|br^k$YwN0qFp6xz#P}n@@=?UY~85*BP}5Kb<}4W!y~M zp}QaNS(sTksou$U+UcLMq#zbv#dP-(#6-p-=sgW~-RkbIy5w8a8$1^*Fml5C{RW+Sj3qI=38f!IT z_I4x%Y#vCdG)Su88})S}b``ACLeC8=`wyXuucil#JSq2g$gQP)S9E0K$5~i>r3aIy zb|Q}-RR`+G$*c5cA8xf#%daC)6~7BvyI2V6%NP83C?paqDlL-3C6t+$u7eANzRhI{ z_h^G?_XrcWR9RBVPl_}T>SM{V(U0$RRoHq@ zdbzM>jEQEyF*%dH{9=(I;pv6GOg_|KpKGw7D2UFx-F%$6gQu9IDxK}d6AR0GE;Lfp zanj9pb#OGX^+@e6h>z&D#}tcO?8K+SLWS^vmQ;qgg-yQe;t|i6`sMB*twoocx?uNi z8vhAr?$US?h}sVsS4462t^mb1w1ezo@B!IKcfD!WIoFg<8m; z2(!wU!|vtnyxfQPp~r=W3<@ZZZ>2>kN7hrBF6Qraup$yQQ?3W=3WHQdYkW_gWxOTN z2|*rv4|zS1$XgeDRwrHxV^vfh1>K0N$VtfYu=_M;BJRg;p`p9x z;ya_)98^lMmjT0KJOhL9r}Z&k=S>hM1lFOqseJGc$_4feGxrioIZ4TbF4hpAKB`aXUXu%CYh1fJcBg4{=jelWi^x1R{}FK$!c)g zjAL)po-jpxc=r$G^^RU;vC9@&6Kz9>Gp0bUo=>&TmW-t?$7?IyWn_i=IX@1xDxd{T zkIHIakYf@!Dcui_N*lZ~NBOJBnLp2vGSI+S>!pwPM|v)fT3$@gW0n8u27Gs)7H6&2`dQs0&o1$^Ysyj?jUDv7j6b({*^E5;Z>s)!_+hwE%U z8A4o3>!?Qf0hq^V2l&a3cbZFL5AOXa&mh7y!t*(Vm{ z;Kc-utR*DSIFpPDTQ%R$AtS~SZc8T`6gqW9J6QyQEcdcrj+HaOd4RzsRunArAsVF+^VO=?u`4MS(%jP(F>VgrQa&R?2|sB@($LJZWvWz;MUu#!G3}0DB#t z%gcQ?Mlmeh^N2sB=Ysi9uig=sZ>eHEY?^QW33^}zZ2#i1`|@o1h@qbaD8HO*1|z4Z N+k2n#`0B63{{UIVm}>w4 literal 0 HcmV?d00001