Zero downtime password rotations

README.md

@@ -25,7 +25,8 @@ PostgreSQL pooler and proxy (like PgBouncer) with support for sharding, load bal
 | Sharding using comments parsing/Regex | **Experimental** | Clients can include shard information (sharding key, shard ID) in the query comments. |
 | Automatic sharding | **Experimental** | PgCat can parse queries, detect sharding keys automatically, and route queries to the correct shard. |
 | Mirroring | **Experimental** | Mirror queries between multiple databases in order to test servers with realistic production traffic. |
-| Auth passthrough | **Experimental** | MD5 password authentication can be configured to use an `auth_query` so no cleartext passwords are needed in the config file.                         |
+| Auth passthrough | **Experimental** | MD5 password authentication can be configured to use an `auth_query` so no cleartext passwords are needed in the config file. |
+| Password rotation | **Experimental** | Allows to rotate passwords without downtime or using third-party tools to manage Postgres authentication. |
 
 
 ## Status
@@ -244,6 +245,12 @@ The config can be reloaded by sending a `kill -s SIGHUP` to the process or by qu
 
 Mirroring allows to route queries to multiple databases at the same time. This is useful for prewarning replicas before placing them into the active configuration, or for testing different versions of Postgres with live traffic.
 
+### Password rotation
+
+Password rotation allows to specify multiple passwords for a user, so they can connect to PgCat with multiple credentials. This allows distributed applications to change their configuration (connection strings) gradually and for PgCat to monitor their progression in admin statistics. Once the new secret is deployed everywhere, the old one can be removed from PgCat.
+
+This also decouples server passwords from client passwords, allowing to change one without necessarily changing the other.
+
 ## License
 
 PgCat is free and open source, released under the MIT license.

dev/docker-compose.yaml

@@ -64,7 +64,7 @@ services:
       <<: *common-env-pg
       POSTGRES_INITDB_ARGS: --auth-local=md5 --auth-host=md5 --auth=md5
       PGPORT: 10432
-    command: ["postgres", "-p", "5432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"]
+    command: ["postgres", "-p", "10432", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "pg_stat_statements.track=all", "-c", "pg_stat_statements.max=100000"]
 
   toxiproxy:
     build: .

pgcat.toml

@@ -58,9 +58,9 @@ tcp_keepalives_count = 5
 tcp_keepalives_interval = 5
 
 # Path to TLS Certficate file to use for TLS connections
-# tls_certificate = "server.cert"
+# tls_certificate = ".circleci/server.cert"
 # Path to TLS private key file to use for TLS connections
-# tls_private_key = "server.key"
+# tls_private_key = ".circleci/server.key"
 
 # User name to access the virtual administrative database (pgbouncer or pgcat)
 # Connecting to that database allows running commands like `SHOW POOLS`, `SHOW DATABASES`, etc..
@@ -122,13 +122,21 @@ idle_timeout = 40000
 # Connect timeout can be overwritten in the pool
 connect_timeout = 3000
 
+# auth_query = "SELECT * FROM public.user_lookup('$1')"
+# auth_query_user = "postgres"
+# auth_query_password = "postgres"
+
 # User configs are structured as pool.<pool_name>.users.<user_index>
 # This secion holds the credentials for users that may connect to this cluster
 [pools.sharded_db.users.0]
 # Postgresql username
 username = "sharding_user"
 # Postgresql password
 password = "sharding_user"
+
+# # Passwords the client can use to connect. Useful for password rotations.
+# secrets = [ "secret_one", "secret_two" ]
+
 # Maximum number of server connections that can be established for this user
 # The maximum number of connection from a single Pgcat process to any database in the cluster
 # is the sum of pool_size across all users.

src/admin.rs

@@ -259,6 +259,7 @@ where
     let columns = vec![
         ("database", DataType::Text),
         ("user", DataType::Text),
+        ("secret", DataType::Text),
         ("pool_mode", DataType::Text),
         ("cl_idle", DataType::Numeric),
         ("cl_active", DataType::Numeric),
@@ -276,10 +277,11 @@ where
     let mut res = BytesMut::new();
     res.put(row_description(&columns));
 
-    for ((_user_pool, _pool), pool_stats) in all_pool_stats {
+    for (_, pool_stats) in all_pool_stats {
         let mut row = vec![
             pool_stats.database(),
             pool_stats.user(),
+            pool_stats.redacted_secret(),
             pool_stats.pool_mode().to_string(),
         ];
         pool_stats.populate_row(&mut row);
@@ -780,7 +782,7 @@ where
         let database = parts[0];
         let user = parts[1];
 
-        match get_pool(database, user) {
+        match get_pool(database, user, None) {
             Some(pool) => {
                 pool.pause();
 
@@ -827,7 +829,7 @@ where
         let database = parts[0];
         let user = parts[1];
 
-        match get_pool(database, user) {
+        match get_pool(database, user, None) {
             Some(pool) => {
                 pool.resume();
 
@@ -895,13 +897,20 @@ where
     res.put(row_description(&vec![
         ("name", DataType::Text),
         ("pool_mode", DataType::Text),
+        ("secret", DataType::Text),
     ]));
 
     for (user_pool, pool) in get_all_pools() {
         let pool_config = &pool.settings;
+        let redacted_secret = match user_pool.secret {
+            Some(secret) => format!("****{}", &secret[secret.len() - 4..]),
+            None => "<no secret>".to_string(),
+        };
+
         res.put(data_row(&vec![
             user_pool.user.clone(),
             pool_config.pool_mode.to_string(),
+            redacted_secret,
         ]));
     }