diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 834b87474..000000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.github/workflows/provision.yml b/.github/workflows/provision.yml deleted file mode 100644 index 9e67c6a60..000000000 --- a/.github/workflows/provision.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: CD - -on: - push: - branches: [ master ] - -jobs: - provision: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Install essentials - run: | - sudo apt-get update -y - sudo apt-get install software-properties-common -y - sudo apt-get install unzip -y - sudo apt-get install git -y - - - name: Install Packer - run: | - export VER="1.5.5" - wget https://releases.hashicorp.com/packer/${VER}/packer_${VER}_linux_amd64.zip - sudo unzip packer_${VER}_linux_amd64.zip - sudo mv packer /usr/local/bin - - - name: Install Ansible - run: | - sudo apt-add-repository --yes ppa:ansible/ansible - sudo apt-get install ansible -y - ansible --version - - - name: Install Ansible role - run: | - sudo ansible-galaxy install anxs.postgresql -r ansible/install_roles.yml --force -vvv - ansible-galaxy list anxs.postgresql - - - name: Build Digital Ocean Snapshot - run: | - export REGION=sgp1 - export SNAPSHOT_REGIONS="nyc1,nyc3,ams3,sfo2,sfo3,sgp1,lon1,fra1,tor1" - sudo packer build \ - -var "do_token=${{ secrets.DO_TOKEN }}" \ - -var "region=$REGION" \ - -var "snapshot_regions=$SNAPSHOT_REGIONS" \ - digitalOcean.json - - # - name: Build Digital Ocean Snapshot for Marketplace - # run: | - # export REGION=sgp1 - # export IMAGE_NAME="supabase-supabasepostgres-18-04" - # export ARGS="--tags,update" - # sudo packer build \ - # -var "do_token=${{ secrets.DO_TOKEN }}" \ - # -var "region=$REGION" \ - # -var "image_name=$IMAGE_NAME" \ - # -var "ansible_arguments=$ARGS" \ - # digitalOcean.json diff --git a/.gitignore b/.gitignore index e69de29bb..8c1f8fabd 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,3 @@ +.DS_Store +.python-version +venv/ diff --git a/README.md b/README.md index 85bc29c7b..fad189412 100644 --- a/README.md +++ b/README.md @@ -2,21 +2,44 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to modify Postgres, but to provide some of the most common extensions with a one-click install. -## Features - -- ✅ Postgres [12](https://www.postgresql.org/about/news/1976/). Includes [generated columns](https://www.postgresql.org/docs/12/ddl-generated-columns.html) and [JSON path](https://www.postgresql.org/docs/12/functions-json.html#FUNCTIONS-SQLJSON-PATH) support. -- ✅ Ubuntu 18.04 (Bionic). -- ✅ [pg-contrib-12](https://www.postgresql.org/docs/12/contrib.html). Because everyone should enable `pg_stat_statements`. +## Primary Features +- ✅ Postgres [13](https://www.postgresql.org/about/news/postgresql-13-released-2077/). +- ✅ Ubuntu 20.04 (Focal Fossa). - ✅ [wal_level](https://www.postgresql.org/docs/current/runtime-config-wal.html) = logical and [max_replication_slots](https://www.postgresql.org/docs/current/runtime-config-replication.html) = 5. Ready for replication. -- ✅ [PostGIS](https://postgis.net/). Postgres' most popular extension - support for geographic objects. -- ✅ [pgTAP](https://pgtap.org/). Unit Testing for Postgres. -- ✅ [pgAudit](https://www.pgaudit.org/). Generate highly compliant audit logs. -- ✅ [pgjwt](https://github.com/michelp/pgjwt). Generate JSON Web Tokens (JWT) in Postgres. -- ✅ [pgsql-http](https://github.com/pramsey/pgsql-http). HTTP client for Postgres. -- ✅ [plpgsql_check](https://github.com/okbob/plpgsql_check). Linter tool for PL/pgSQL. -- ✅ [plv8](https://github.com/plv8/plv8). Write in Javascript functions in Postgres. -- ✅ [plpython3u](https://www.postgresql.org/docs/current/plpython-python23.html). Python3 enabled by default. Write in Python functions in Postgres. -- ✅ [PL/Java](https://github.com/tada/pljaval). Write in Java functions in Postgres. +- ✅ [Large Systems Extensions](https://github.com/aws/aws-graviton-getting-started#building-for-graviton-and-graviton2). Enabled for ARM images. + +## Extensions +| Extension | Description | +| ------------- | ------------- | +| [Postgres contrib modules](https://www.postgresql.org/docs/current/contrib.html) | Because everyone should enable `pg_stat_statements`. | +| [PostGIS](https://postgis.net/) | Postgres' most popular extension - support for geographic objects. | +| [pgRouting](https://pgrouting.org/) | Extension of PostGIS - provides geospatial routing functionalities. | +| [pgTAP](https://pgtap.org/) | Unit Testing for Postgres. | +| [pg_cron](https://github.com/citusdata/pg_cron) | Run CRON jobs inside Postgres. | +| [pgAudit](https://www.pgaudit.org/) | Generate highly compliant audit logs. | +| [pgjwt](https://github.com/michelp/pgjwt) | Generate JSON Web Tokens (JWT) in Postgres. | +| [pgsql-http](https://github.com/pramsey/pgsql-http) | HTTP client for Postgres. | +| [plpgsql_check](https://github.com/okbob/plpgsql_check) | Linter tool for PL/pgSQL. | +| [pg-safeupdate](https://github.com/eradman/pg-safeupdate) | Protect your data from accidental updates or deletes. | +| [wal2json](https://github.com/eulerto/wal2json) | JSON output plugin for logical replication decoding. | +| [PL/Java](https://github.com/tada/pljava) | Write in Java functions in Postgres. | +| [plv8](https://github.com/plv8/plv8) | Write in Javascript functions in Postgres. | + +Can't find your favorite extension? Suggest for it to be added into future versions [here](https://github.com/supabase/supabase/discussions/679)! + +## Enhanced Security +Aside from having [ufw](https://help.ubuntu.com/community/UFW),[fail2ban](https://www.fail2ban.org/wiki/index.php/Main_Page), and [unattended-upgrades](https://wiki.debian.org/UnattendedUpgrades) installed, we also have the following enhancements in place: +| Enhancement | Description | +| ------------- | ------------- | +| [fail2ban filter](https://github.com/supabase/postgres/blob/develop/ansible/files/fail2ban_config/filter-postgresql.conf.j2) for PostgreSQL access | Monitors for brute force attempts over at port `5432`. | +| [fail2ban filter](https://github.com/supabase/postgres/blob/develop/ansible/files/fail2ban_config/filter-pgbouncer.conf.j2) for PgBouncer access | Monitors for brute force attempts over at port `6543`. | + +## Additional Goodies +| Goodie | Description | +| ------------- | ------------- | +| [PgBouncer](https://postgis.net/) | Set up Connection Pooling. | +| [PostgREST](https://postgrest.org/en/stable/) | Instantly transform your database into an RESTful API. | +| [WAL-G](https://github.com/wal-g/wal-g#wal-g) | Tool for physical database backup and recovery. | ## Install @@ -26,13 +49,39 @@ See all installation instructions in the [repo wiki](https://github.com/supabase [![Digital Ocean](https://github.com/supabase/postgres/blob/master/docs/img/digital-ocean.png)](https://github.com/supabase/postgres/wiki/Digital-Ocean) [![AWS](https://github.com/supabase/postgres/blob/master/docs/img/aws.png)](https://github.com/supabase/postgres/wiki/AWS-EC2) +### Marketplace Images +| | Postgres & Extensions | PgBouncer | PostgREST | WAL-G | +|---|:---:|:---:|:---:|:---:| +| Supabase Postgres | ✔️ | ❌ | ❌ | ✔️ | +| Supabase Postgres: PgBouncer Bundle | ✔️ | ✔️ | ❌ | ✔️ | +| Supabase Postgres: PostgREST Bundle | ✔️ | ❌ | ✔️ | ✔️ | +| Supabase Postgres: Complete Bundle | ✔️ | ✔️ | ✔️ | ✔️ | + +#### Availability +| | AWS ARM | AWS x86 | Digital Ocean x86 | +|---|:---:|:---:|:---:| +| Supabase Postgres | Coming Soon | Coming Soon | Coming Soon | +| Supabase Postgres: PgBouncer Bundle | Coming Soon | Coming Soon | Coming Soon | +| Supabase Postgres: PostgREST Bundle | Coming Soon | Coming Soon | Coming Soon | +| Supabase Postgres: Complete Bundle | Coming Soon | Coming Soon | Coming Soon | + +### Quick Build + +```bash +$ time packer build -timestamp-ui \ + --var "aws_access_key=" \ + --var "aws_secret_key=" \ + --var "ami_regions=" \ + amazon-arm.json +``` + ## Motivation -After talking to a lot of techies, we've found that most believe Postgres is the best (operational) database but they *still* choose other databases. This is overwhelmingly because "the other one was quicker/easier". Our goal is to make it fast and simple to get started with Postgres, so that we never hear that excuse again. +After talking to a lot of techies, we've found that most believe Postgres is the best (operational) database but they _still_ choose other databases. This is overwhelmingly because "the other one was quicker/easier". Our goal is to make it fast and simple to get started with Postgres, so that we never hear that excuse again. Our secondary goal is to show off a few of Postgres' most exciting features. This is to convince new developers to choose it over other database (a decision we hope they'll appreciate once they start scaling). -Finally, this is the same build we offer at [Supabase](https://supabase.io), and everything we do is opensource. This repo makes it easy to *install* Postgres, Supabase makes it easy to *use* Postgres. +Finally, this is the same build we offer at [Supabase](https://supabase.io), and everything we do is opensource. This repo makes it easy to _install_ Postgres, Supabase makes it easy to _use_ Postgres. ## Roadmap @@ -43,3 +92,9 @@ Finally, this is the same build we offer at [Supabase](https://supabase.io), and ## License [The PostgreSQL License](https://opensource.org/licenses/postgresql). We realize that licensing is tricky since we are bundling all the various plugins. If we have infringed on any license, let us know and we will make the necessary changes (or remove that extension from this repo). + +## Sponsors + +We are building the features of Firebase using enterprise-grade, open source products. We support existing communities wherever possible, and if the products don’t exist we build them and open source them ourselves. + +[![New Sponsor](https://user-images.githubusercontent.com/10214025/90518111-e74bbb00-e198-11ea-8f88-c9e3c1aa4b5b.png)](https://github.com/sponsors/supabase) diff --git a/amazon-arm.json b/amazon-arm.json new file mode 100644 index 000000000..4a1b2acb3 --- /dev/null +++ b/amazon-arm.json @@ -0,0 +1,60 @@ +{ + "variables": { + "aws_access_key": "", + "aws_secret_key": "", + "region": "ap-northeast-1", + "ami_regions": "eu-central-1,eu-west-1,eu-west-2,ap-south-1,ap-southeast-1,ap-southeast-2,us-west-1,us-east-1,ca-central-1,sa-east-1,ap-northeast-1", + "ami": "ami-076d8ebdd0e1ec091", + "ami_name": "supabase-postgres-13.3.0", + "environment": "prod", + "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,--skip-tags,install-pgbouncer,--skip-tags,install-supabase-internal" + }, + "builders": [ + { + "type": "amazon-ebs", + "access_key": "{{user `aws_access_key`}}", + "secret_key": "{{user `aws_secret_key`}}", + "region": "{{user `region`}}", + "ami_regions": "{{user `ami_regions`}}", + "source_ami": "{{user `ami`}}", + "instance_type": "r6g.2xlarge", + "ssh_username": "ubuntu", + "ami_name": "{{user `ami_name`}}", + "tags": { + "environment": "{{user `environment`}}", + "appType": "postgres" + }, + "launch_block_device_mappings": [ + { + "device_name": "/dev/sda1", + "volume_size": 16, + "volume_type": "gp2", + "delete_on_termination": true + } + ] + } + ], + "provisioners": [ + { + "type": "shell", + "inline": [ + "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done" + ] + }, + { + "type": "ansible", + "user": "ubuntu", + "playbook_file": "ansible/playbook.yml", + "extra_arguments": "{{user `ansible_arguments`}}" + }, + { + "execute_command": "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'", + "type": "shell", + "scripts": [ + "scripts/02-credentials_cleanup.sh", + "scripts/90-cleanup.sh", + "scripts/91-log_cleanup.sh" + ] + } + ] +} diff --git a/amazon.json b/amazon.json index 69dee7aa8..0cc4ef860 100644 --- a/amazon.json +++ b/amazon.json @@ -2,31 +2,45 @@ "variables": { "aws_access_key": "", "aws_secret_key": "", - "region": "ap-southeast-1", - "ami_regions": "ap-southeast-1", - "ami": "ami-0f7719e8b7ba25c61", - "ansible_arguments": "--skip-tags,update-only" + "region": "af-south-1", + "ami_regions": "af-south-1", + "ami": "ami-08a4b40f2fe1e4b35", + "ami_name": "supabase-postgres-13.3.0.4", + "environment": "prod", + "ansible_arguments": "--skip-tags,update-only,--skip-tags,install-postgrest,--skip-tags,install-pgbouncer,--skip-tags,install-supabase-internal" }, - "builders": [{ - "type": "amazon-ebs", - "access_key": "{{user `aws_access_key`}}", - "secret_key": "{{user `aws_secret_key`}}", - "region": "{{user `region`}}", - "ami_regions": "{{user `ami_regions`}}", - "source_ami": "{{user `ami`}}", - "instance_type": "m5.2xlarge", - "ssh_username": "ubuntu", - "ami_name": "supabase-postgres-0.13.0", - "launch_block_device_mappings": [ - { - "device_name": "/dev/sda1", - "volume_size": 16, - "volume_type": "gp2", - "delete_on_termination": true - } - ] - }], + "builders": [ + { + "type": "amazon-ebs", + "access_key": "{{user `aws_access_key`}}", + "secret_key": "{{user `aws_secret_key`}}", + "region": "{{user `region`}}", + "ami_regions": "{{user `ami_regions`}}", + "source_ami": "{{user `ami`}}", + "instance_type": "m5.2xlarge", + "ssh_username": "ubuntu", + "ami_name": "{{user `ami_name`}}", + "tags": { + "environment": "{{user `environment`}}", + "appType": "postgres" + }, + "launch_block_device_mappings": [ + { + "device_name": "/dev/sda1", + "volume_size": 16, + "volume_type": "gp2", + "delete_on_termination": true + } + ] + } + ], "provisioners": [ + { + "type": "shell", + "inline": [ + "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done" + ] + }, { "type": "ansible", "user": "ubuntu", @@ -37,11 +51,9 @@ "execute_command": "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'", "type": "shell", "scripts": [ - "scripts/01-test", "scripts/02-credentials_cleanup.sh", "scripts/90-cleanup.sh", - "scripts/91-log_cleanup.sh", - "scripts/99-img_check.sh" + "scripts/91-log_cleanup.sh" ] } ] diff --git a/ansible/files/docker_mnt/init.sh b/ansible/files/docker_mnt/init.sh new file mode 100644 index 000000000..fd12bbe4a --- /dev/null +++ b/ansible/files/docker_mnt/init.sh @@ -0,0 +1,3 @@ +cat /etc/postgresql/postgresql.conf > $PGDATA/postgresql.conf +echo "host replication $POSTGRES_USER 0.0.0.0/0 trust" >> $PGDATA/pg_hba.conf +echo "host all all 127.0.0.1/32 trust" >> $PGDATA/pg_hba.conf \ No newline at end of file diff --git a/ansible/files/fail2ban_config/filter-pgbouncer.conf.j2 b/ansible/files/fail2ban_config/filter-pgbouncer.conf.j2 new file mode 100644 index 000000000..50326da51 --- /dev/null +++ b/ansible/files/fail2ban_config/filter-pgbouncer.conf.j2 @@ -0,0 +1,2 @@ +[Definition] +failregex = ^.+@:.+error: password authentication failed$ \ No newline at end of file diff --git a/ansible/files/fail2ban_config/filter-postgresql.conf.j2 b/ansible/files/fail2ban_config/filter-postgresql.conf.j2 new file mode 100644 index 000000000..fd0895aee --- /dev/null +++ b/ansible/files/fail2ban_config/filter-postgresql.conf.j2 @@ -0,0 +1,3 @@ +[Definition] +failregex = ^.*,.*,.*,.*,":.*password authentication failed for user.*$ +ignoreregex = ^.*,.*,.*,.*,"127\.0\.0\.1.*password authentication failed for user.*$ \ No newline at end of file diff --git a/ansible/files/fail2ban_config/jail-pgbouncer.conf.j2 b/ansible/files/fail2ban_config/jail-pgbouncer.conf.j2 new file mode 100644 index 000000000..77c5530dc --- /dev/null +++ b/ansible/files/fail2ban_config/jail-pgbouncer.conf.j2 @@ -0,0 +1,7 @@ +[pgbouncer] +enabled = true +port = 6543 +protocol = tcp +filter = pgbouncer +logpath = /var/log/pgbouncer.log +maxretry = 3 \ No newline at end of file diff --git a/ansible/files/fail2ban_config/jail-postgresql.conf.j2 b/ansible/files/fail2ban_config/jail-postgresql.conf.j2 new file mode 100644 index 000000000..516f5320b --- /dev/null +++ b/ansible/files/fail2ban_config/jail-postgresql.conf.j2 @@ -0,0 +1,7 @@ +[postgresql] +enabled = true +port = 5432 +protocol = tcp +filter = postgresql +logpath = /var/lib/postgresql/data/pg_log/postgresql.csv +maxretry = 3 \ No newline at end of file diff --git a/ansible/files/logrotate-postgres b/ansible/files/logrotate-postgres new file mode 100644 index 000000000..3266dbdd8 --- /dev/null +++ b/ansible/files/logrotate-postgres @@ -0,0 +1,9 @@ +/var/lib/postgresql/data/pg_log/postgresql.log { + size 50M + rotate 3 + copytruncate + delaycompress + compress + notifempty + missingok +} diff --git a/ansible/files/node_exporter.service.j2 b/ansible/files/node_exporter.service.j2 new file mode 100644 index 000000000..4af7195da --- /dev/null +++ b/ansible/files/node_exporter.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Node Exporter +After=network-online.target + +[Service] +Type=simple +ExecStart=/opt/node_exporter/node_exporter --web.disable-exporter-metrics --collector.disable-defaults {% for collector in collectors %} --collector.{{ collector }} {% endfor %} + +User=root +StandardOutput=file:/var/log/node_exporter.stdout +StandardError=file:/var/log/node_exporter.error +Restart=on-failure +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/files/pgbouncer_config/pgbouncer.ini.j2 b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 new file mode 100644 index 000000000..c0c5dd0f6 --- /dev/null +++ b/ansible/files/pgbouncer_config/pgbouncer.ini.j2 @@ -0,0 +1,360 @@ +;;; +;;; PgBouncer configuration file +;;; + +;; database name = connect string +;; +;; connect string params: +;; dbname= host= port= user= password= auth_user= +;; client_encoding= datestyle= timezone= +;; pool_size= reserve_pool= max_db_connections= +;; pool_mode= connect_query= application_name= +[databases] +* = host=localhost auth_user=pgbouncer + +;; foodb over Unix socket +;foodb = + +;; redirect bardb to bazdb on localhost +;bardb = host=localhost dbname=bazdb + +;; access to dest database will go with single user +;forcedb = host=localhost port=300 user=baz password=foo client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1' + +;; use custom pool sizes +;nondefaultdb = pool_size=50 reserve_pool=10 + +;; use auth_user with auth_query if user not present in auth_file +;; auth_user must exist in auth_file +; foodb = auth_user=bar + +;; fallback connect string +;* = host=testserver + +;; User-specific configuration +[users] + +;user1 = pool_mode=transaction max_user_connections=10 + +;; Configuration section +[pgbouncer] + +;;; +;;; Administrative settings +;;; + +logfile = /var/log/pgbouncer.log +pidfile = /var/run/pgbouncer/pgbouncer.pid + +;;; +;;; Where to wait for clients +;;; + +;; IP address or * which means all IPs +listen_addr = * +listen_port = 6543 + +;; Unix socket is also used for -R. +;; On Debian it should be /var/run/postgresql +unix_socket_dir = /tmp +;unix_socket_mode = 0777 +;unix_socket_group = + +;;; +;;; TLS settings for accepting clients +;;; + +;; disable, allow, require, verify-ca, verify-full +;client_tls_sslmode = disable + +;; Path to file that contains trusted CA certs +;client_tls_ca_file = + +;; Private key and cert to present to clients. +;; Required for accepting TLS connections from clients. +;client_tls_key_file = +;client_tls_cert_file = + +;; fast, normal, secure, legacy, +;client_tls_ciphers = fast + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 +;client_tls_protocols = secure + +;; none, auto, legacy +;client_tls_dheparams = auto + +;; none, auto, +;client_tls_ecdhcurve = auto + +;;; +;;; TLS settings for connecting to backend databases +;;; + +;; disable, allow, require, verify-ca, verify-full +;server_tls_sslmode = disable + +;; Path to that contains trusted CA certs +;server_tls_ca_file = + +;; Private key and cert to present to backend. +;; Needed only if backend server require client cert. +;server_tls_key_file = +;server_tls_cert_file = + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 +;server_tls_protocols = secure + +;; fast, normal, secure, legacy, +;server_tls_ciphers = fast + +;;; +;;; Authentication settings +;;; + +;; any, trust, plain, md5, cert, hba, pam +auth_type = md5 +auth_file = /etc/pgbouncer/userlist.txt + +;; Path to HBA-style auth config +;auth_hba_file = + +;; Query to use to fetch password from database. Result +;; must have 2 columns - username and password hash. +auth_query = SELECT * FROM pgbouncer.get_auth($1) + +;;; +;;; Users allowed into database 'pgbouncer' +;;; + +;; comma-separated list of users who are allowed to change settings +admin_users = pgbouncer + +;; comma-separated list of users who are just allowed to use SHOW command +stats_users = pgbouncer + +;;; +;;; Pooler personality questions +;;; + +;; When server connection is released back to pool: +;; session - after client disconnects (default) +;; transaction - after transaction finishes +;; statement - after statement finishes +pool_mode = transaction + +;; Query for cleaning connection immediately after releasing from +;; client. No need to put ROLLBACK here, pgbouncer does not reuse +;; connections where transaction is left open. +;server_reset_query = DISCARD ALL + +;; Whether server_reset_query should run in all pooling modes. If it +;; is off, server_reset_query is used only for session-pooling. +;server_reset_query_always = 0 + +;; Comma-separated list of parameters to ignore when given in startup +;; packet. Newer JDBC versions require the extra_float_digits here. +ignore_startup_parameters = extra_float_digits + +;; When taking idle server into use, this query is run first. +;server_check_query = select 1 + +;; If server was used more recently that this many seconds ago, +; skip the check query. Value 0 may or may not run in immediately. +;server_check_delay = 30 + +;; Close servers in session pooling mode after a RECONNECT, RELOAD, +;; etc. when they are idle instead of at the end of the session. +;server_fast_close = 0 + +;; Use as application_name on server. +;application_name_add_host = 0 + +;; Period for updating aggregated stats. +;stats_period = 60 + +;;; +;;; Connection limits +;;; + +;; Total number of clients that can connect +;max_client_conn = 100 + +;; Default pool size. 20 is good number when transaction pooling +;; is in use, in session pooling it needs to be the number of +;; max clients you want to handle at any moment +default_pool_size = 15 + +;; Minimum number of server connections to keep in pool. +;min_pool_size = 0 + +; how many additional connection to allow in case of trouble +;reserve_pool_size = 0 + +;; If a clients needs to wait more than this many seconds, use reserve +;; pool. +;reserve_pool_timeout = 5 + +;; Maximum number of server connections for a database +;max_db_connections = 0 + +;; Maximum number of server connections for a user +;max_user_connections = 0 + +;; If off, then server connections are reused in LIFO manner +;server_round_robin = 0 + +;;; +;;; Logging +;;; + +;; Syslog settings +;syslog = 0 +;syslog_facility = daemon +;syslog_ident = pgbouncer + +;; log if client connects or server connection is made +;log_connections = 1 + +;; log if and why connection was closed +;log_disconnections = 1 + +;; log error messages pooler sends to clients +;log_pooler_errors = 1 + +;; write aggregated stats into log +;log_stats = 1 + +;; Logging verbosity. Same as -v switch on command line. +;verbose = 0 + +;;; +;;; Timeouts +;;; + +;; Close server connection if its been connected longer. +;server_lifetime = 3600 + +;; Close server connection if its not been used in this time. Allows +;; to clean unnecessary connections from pool after peak. +;server_idle_timeout = 600 + +;; Cancel connection attempt if server does not answer takes longer. +;server_connect_timeout = 15 + +;; If server login failed (server_connect_timeout or auth failure) +;; then wait this many second. +;server_login_retry = 15 + +;; Dangerous. Server connection is closed if query does not return in +;; this time. Should be used to survive network problems, _not_ as +;; statement_timeout. (default: 0) +;query_timeout = 0 + +;; Dangerous. Client connection is closed if the query is not +;; assigned to a server in this time. Should be used to limit the +;; number of queued queries in case of a database or network +;; failure. (default: 120) +;query_wait_timeout = 120 + +;; Dangerous. Client connection is closed if no activity in this +;; time. Should be used to survive network problems. (default: 0) +;client_idle_timeout = 0 + +;; Disconnect clients who have not managed to log in after connecting +;; in this many seconds. +;client_login_timeout = 60 + +;; Clean automatically created database entries (via "*") if they stay +;; unused in this many seconds. +; autodb_idle_timeout = 3600 + +;; Close connections which are in "IDLE in transaction" state longer +;; than this many seconds. +;idle_transaction_timeout = 0 + +;; How long SUSPEND/-R waits for buffer flush before closing +;; connection. +;suspend_timeout = 10 + +;;; +;;; Low-level tuning options +;;; + +;; buffer for streaming packets +;pkt_buf = 4096 + +;; man 2 listen +;listen_backlog = 128 + +;; Max number pkt_buf to process in one event loop. +;sbuf_loopcnt = 5 + +;; Maximum PostgreSQL protocol packet size. +;max_packet_size = 2147483647 + +;; Set SO_REUSEPORT socket option +;so_reuseport = 0 + +;; networking options, for info: man 7 tcp + +;; Linux: Notify program about new connection only if there is also +;; data received. (Seconds to wait.) On Linux the default is 45, on +;; other OS'es 0. +;tcp_defer_accept = 0 + +;; In-kernel buffer size (Linux default: 4096) +;tcp_socket_buffer = 0 + +;; whether tcp keepalive should be turned on (0/1) +;tcp_keepalive = 1 + +;; The following options are Linux-specific. They also require +;; tcp_keepalive=1. + +;; Count of keepalive packets +;tcp_keepcnt = 0 + +;; How long the connection can be idle before sending keepalive +;; packets +;tcp_keepidle = 0 + +;; The time between individual keepalive probes +;tcp_keepintvl = 0 + +;; How long may transmitted data remain unacknowledged before TCP +;; connection is closed (in milliseconds) +;tcp_user_timeout = 0 + +;; DNS lookup caching time +;dns_max_ttl = 15 + +;; DNS zone SOA lookup period +;dns_zone_check_period = 0 + +;; DNS negative result caching time +;dns_nxdomain_ttl = 15 + +;; Custom resolv.conf file, to set custom DNS servers or other options +;; (default: empty = use OS settings) +;resolv_conf = /etc/pgbouncer/resolv.conf + +;;; +;;; Random stuff +;;; + +;; Hackish security feature. Helps against SQL injection: when PQexec +;; is disabled, multi-statement cannot be made. +;disable_pqexec = 0 + +;; Config file to use for next RELOAD/SIGHUP +;; By default contains config file from command line. +;conffile + +;; Windows service name to register as. job_name is alias for +;; service_name, used by some Skytools scripts. +;service_name = pgbouncer +;job_name = pgbouncer + +;; Read additional config from other file +;%include /etc/pgbouncer/pgbouncer-other.ini diff --git a/ansible/files/pgbouncer_config/pgbouncer.service.j2 b/ansible/files/pgbouncer_config/pgbouncer.service.j2 new file mode 100644 index 000000000..96273cb69 --- /dev/null +++ b/ansible/files/pgbouncer_config/pgbouncer.service.j2 @@ -0,0 +1,40 @@ +# Example systemd service unit for PgBouncer +# +# - Adjust the paths in ExecStart for your installation. +# +# - The User setting requires careful consideration. PgBouncer needs +# to be able to place a Unix-domain socket file where PostgreSQL +# clients will look for it. In the olden days, this was in /tmp, +# but systems using systemd now prefer something like +# /var/run/postgresql/. But then some systems also lock down that +# directory so that only the postgres user can write to it. That +# means you need to either +# +# - run PgBouncer as the postgres user, or +# +# - create a separate user and add it to the postgres group and +# make /var/run/postgresql/ group-writable, or +# +# - use systemd to create the sockets; see pgbouncer.socket nearby. +# +# For packagers and deployment systems, this requires some +# coordination between the PgBouncer and the PostgreSQL +# packages/components. +# +[Unit] +Description=connection pooler for PostgreSQL +Documentation=man:pgbouncer(1) +Documentation=https://www.pgbouncer.org/ +After=network.target +#Requires=pgbouncer.socket + +[Service] +Type=notify +User=postgres +ExecStart=/usr/local/bin/pgbouncer /etc/pgbouncer/pgbouncer.ini +ExecReload=/bin/kill -HUP $MAINPID +KillSignal=SIGINT +#LimitNOFILE=1024 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql b/ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql new file mode 100644 index 000000000..bc1342f8d --- /dev/null +++ b/ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql @@ -0,0 +1,18 @@ +CREATE USER pgbouncer; + +CREATE SCHEMA pgbouncer AUTHORIZATION pgbouncer; + +CREATE OR REPLACE FUNCTION pgbouncer.get_auth(p_usename TEXT) +RETURNS TABLE(username TEXT, password TEXT) AS +$$ +BEGIN + RAISE WARNING 'PgBouncer auth request: %', p_usename; + + RETURN QUERY + SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow + WHERE usename = p_usename; +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +REVOKE ALL ON FUNCTION pgbouncer.get_auth(p_usename TEXT) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(p_usename TEXT) TO pgbouncer; diff --git a/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 b/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 new file mode 100644 index 000000000..3889ed294 --- /dev/null +++ b/ansible/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 @@ -0,0 +1,2 @@ +# Directory for PostgreSQL sockets, lockfiles and stats tempfiles +d /run/pgbouncer 2775 postgres postgres - - \ No newline at end of file diff --git a/ansible/files/postgres_exporter.service.j2 b/ansible/files/postgres_exporter.service.j2 new file mode 100644 index 000000000..906e4566d --- /dev/null +++ b/ansible/files/postgres_exporter.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Postgres Exporter + +[Service] +Type=simple +ExecStart=/opt/postgres_exporter/postgres_exporter --auto-discover-databases --disable-settings-metrics --extend.query-path="/opt/postgres_exporter/queries.yml" --disable-default-metrics +User=root +StandardOutput=file:/var/log/postgres_exporter.stdout +StandardError=file:/var/log/postgres_exporter.error +Restart=always +RestartSec=3 +Environment="DATA_SOURCE_URI=localhost/postgres?sslmode=disable" +Environment="DATA_SOURCE_USER=supabase_admin" + +[Install] +WantedBy=multi-user.target diff --git a/ansible/files/postgresql_config/pg_hba.conf.j2 b/ansible/files/postgresql_config/pg_hba.conf.j2 new file mode 100755 index 000000000..1e4c86604 --- /dev/null +++ b/ansible/files/postgresql_config/pg_hba.conf.j2 @@ -0,0 +1,99 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostgssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnogssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain +# socket, "host" is either a plain or SSL-encrypted TCP/IP socket, +# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a +# non-SSL TCP/IP socket. Similarly, "hostgssenc" uses a +# GSSAPI-encrypted TCP/IP socket, while "hostnogssenc" uses a +# non-GSSAPI socket. +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", +# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". +# Note that "password" sends passwords in clear text; "md5" or +# "scram-sha-256" are preferred since they send encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# "all", "sameuser", "samerole" or "replication" makes the name lose +# its special character, and just match a database or username with +# that name. +# +# This file is read on server startup and when the server receives a +# SIGHUP signal. If you edit the file on a running system, you have to +# SIGHUP the server for the changes to take effect, run "pg_ctl reload", +# or execute "SELECT pg_reload_conf()". +# +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + +# TYPE DATABASE USER ADDRESS METHOD + +# Default: +# "local" is for Unix domain socket connections only +local all all peer +# IPv4 local connections: +host all all 127.0.0.1/32 trust +# IPv6 local connections: +host all all ::1/128 md5 +# Local root Unix user, passwordless access +local all postgres peer map=root_as_postgres +# IPv4 external connections +host all all 0.0.0.0/0 md5 + +# MD5 hashed password hosts + +# Password hosts + +# Trusted hosts + +# User custom \ No newline at end of file diff --git a/ansible/files/postgresql_config/pg_ident.conf.j2 b/ansible/files/postgresql_config/pg_ident.conf.j2 new file mode 100755 index 000000000..a5c8de766 --- /dev/null +++ b/ansible/files/postgresql_config/pg_ident.conf.j2 @@ -0,0 +1,44 @@ +# PostgreSQL User Name Maps +# ========================= +# +# Refer to the PostgreSQL documentation, chapter "Client +# Authentication" for a complete description. A short synopsis +# follows. +# +# This file controls PostgreSQL user name mapping. It maps external +# user names to their corresponding PostgreSQL user names. Records +# are of the form: +# +# MAPNAME SYSTEM-USERNAME PG-USERNAME +# +# (The uppercase quantities must be replaced by actual values.) +# +# MAPNAME is the (otherwise freely chosen) map name that was used in +# pg_hba.conf. SYSTEM-USERNAME is the detected user name of the +# client. PG-USERNAME is the requested PostgreSQL user name. The +# existence of a record specifies that SYSTEM-USERNAME may connect as +# PG-USERNAME. +# +# If SYSTEM-USERNAME starts with a slash (/), it will be treated as a +# regular expression. Optionally this can contain a capture (a +# parenthesized subexpression). The substring matching the capture +# will be substituted for \1 (backslash-one) if present in +# PG-USERNAME. +# +# Multiple maps may be specified in this file and used by pg_hba.conf. +# +# No map names are defined in the default configuration. If all +# system user names and PostgreSQL user names are the same, you don't +# need anything in this file. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can +# use "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- + +# MAPNAME SYSTEM-USERNAME PG-USERNAME +# root is allowed to login as postgres +root_as_postgres postgres postgres diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 new file mode 100644 index 000000000..8b4fec8e2 --- /dev/null +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -0,0 +1,794 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, run "pg_ctl reload", or execute +# "SELECT pg_reload_conf()". Some parameters, which are marked below, +# require a server shutdown and restart to take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: kB = kilobytes Time units: ms = milliseconds +# MB = megabytes s = seconds +# GB = gigabytes min = minutes +# TB = terabytes h = hours +# d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +data_directory = '/var/lib/postgresql/data' # use data in another directory + # (change requires restart) +hba_file = '/etc/postgresql/pg_hba.conf' # host-based authentication file + # (change requires restart) +ident_file = '/etc/postgresql/pg_ident.conf' # ident configuration file + # (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +#external_pid_file = '' # write an extra PID file + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +listen_addresses = '*' # what IP address(es) to listen on; + # comma-separated list of addresses; + # defaults to 'localhost'; use '*' for all + # (change requires restart) +#port = 5432 # (change requires restart) +#max_connections = 100 # (change requires restart) +#superuser_reserved_connections = 3 # (change requires restart) +#unix_socket_directories = '/tmp' # comma-separated list of directories + # (change requires restart) +#unix_socket_group = '' # (change requires restart) +#unix_socket_permissions = 0777 # begin with 0 to use octal notation + # (change requires restart) +#bonjour = off # advertise server via Bonjour + # (change requires restart) +#bonjour_name = '' # defaults to the computer name + # (change requires restart) + +# - TCP settings - +# see "man tcp" for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; + # 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; + # 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; + # 0 selects the system default +#tcp_user_timeout = 0 # TCP_USER_TIMEOUT, in milliseconds; + # 0 selects the system default + +#client_connection_check_interval = 0 # time between checks for client + # disconnection while running queries; + # 0 for never + +# - Authentication - + +authentication_timeout = 1min # 1s-600s +password_encryption = md5 # scram-sha-256 or md5 +db_user_namespace = off + +# GSSAPI using Kerberos +#krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab' +#krb_caseins_users = off + +# - SSL - + +ssl = off +ssl_ca_file = '' +ssl_cert_file = '' +ssl_crl_file = '' +ssl_key_file = '' +ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +ssl_prefer_server_ciphers = on +ssl_ecdh_curve = 'prime256v1' +ssl_min_protocol_version = 'TLSv1.2' +ssl_max_protocol_version = '' +ssl_dh_params_file = '' +ssl_passphrase_command = '' +ssl_passphrase_command_supports_reload = off + + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# - Memory - + +shared_buffers = 128MB # min 128kB + # (change requires restart) +# huge_pages = try # on, off, or try + # (change requires restart) +# huge_page_size = 0 # zero for system default + # (change requires restart) +# temp_buffers = 8MB # min 800kB +# max_prepared_transactions = 0 # zero disables the feature + # (change requires restart) +# Caution: it is not advisable to set max_prepared_transactions nonzero unless +# you actively intend to use prepared transactions. +# work_mem = 4MB # min 64kB +# hash_mem_multiplier = 1.0 # 1-1000.0 multiplier on hash table work_mem +# maintenance_work_mem = 64MB # min 1MB +# autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +# logical_decoding_work_mem = 64MB # min 64kB +# max_stack_depth = 2MB # min 100kB +# shared_memory_type = mmap # the default is the first option + # supported by the operating system: + # mmap + # sysv + # windows + # (change requires restart) +# dynamic_shared_memory_type = posix # the default is the first option + # supported by the operating system: + # posix + # sysv + # windows + # mmap + # (change requires restart) +#min_dynamic_shared_memory = 0MB # (change requires restart) + +# - Disk - + +#temp_file_limit = -1 # limits per-process temp file space + # in kilobytes, or -1 for no limit + +# - Kernel Resources - + +#max_files_per_process = 1000 # min 64 + # (change requires restart) + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-100 milliseconds (0 disables) +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 2 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 1-10000 credits + +# - Background Writer - + +#bgwriter_delay = 200ms # 10-10000ms between rounds +#bgwriter_lru_maxpages = 100 # max buffers written/round, 0 disables +#bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round +#bgwriter_flush_after = 0 # measured in pages, 0 disables + +# - Asynchronous Behavior - + +#backend_flush_after = 0 # measured in pages, 0 disables +#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching +#maintenance_io_concurrency = 10 # 1-1000; 0 disables prefetching +#max_worker_processes = 8 # (change requires restart) +#max_parallel_workers_per_gather = 2 # taken from max_parallel_workers +#max_parallel_maintenance_workers = 2 # taken from max_parallel_workers +#max_parallel_workers = 8 # maximum number of max_worker_processes that + # can be used in parallel operations +#parallel_leader_participation = on +#old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate + # (change requires restart) + + +#------------------------------------------------------------------------------ +# WRITE-AHEAD LOG +#------------------------------------------------------------------------------ + +# - Settings - + +wal_level = logical # minimal, replica, or logical + # (change requires restart) +#fsync = on # flush data to disk for crash safety + # (turning this off can cause + # unrecoverable data corruption) +#synchronous_commit = on # synchronization level; + # off, local, remote_write, remote_apply, or on +#wal_sync_method = fsync # the default is the first option + # supported by the operating system: + # open_datasync + # fdatasync (default on Linux and FreeBSD) + # fsync + # fsync_writethrough + # open_sync +#full_page_writes = on # recover from partial page writes +#wal_log_hints = off # also do full page writes of non-critical updates + # (change requires restart) +#wal_compression = off # enable compression of full-page writes +#wal_init_zero = on # zero-fill new WAL files +#wal_recycle = on # recycle WAL files +#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers + # (change requires restart) +#wal_writer_delay = 200ms # 1-10000 milliseconds +#wal_writer_flush_after = 1MB # measured in pages, 0 disables +#wal_skip_threshold = 2MB + +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_timeout = 5min # range 30s-1d +checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 +checkpoint_flush_after = 256kB # measured in pages, 0 disables +#checkpoint_warning = 30s # 0 disables +#max_wal_size = 1GB +#min_wal_size = 80MB + +# - Archiving - + +#archive_mode = off # enables archiving; off, on, or always + # (change requires restart) +#archive_command = '' # command to use to archive a logfile segment + # placeholders: %p = path of file to archive + # %f = file name only + # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' +#archive_timeout = 0 # force a logfile segment switch after this + # number of seconds; 0 disables + +# - Archive Recovery - + +# These are only used in recovery mode. + +#restore_command = '' # command to use to restore an archived logfile segment + # placeholders: %p = path of file to restore + # %f = file name only + # e.g. 'cp /mnt/server/archivedir/%f %p' +#archive_cleanup_command = '' # command to execute at every restartpoint +#recovery_end_command = '' # command to execute at completion of recovery + +# - Recovery Target - + +# Set these only when performing a targeted recovery. + +#recovery_target = '' # 'immediate' to end recovery as soon as a + # consistent state is reached + # (change requires restart) +#recovery_target_name = '' # the named restore point to which recovery will proceed + # (change requires restart) +#recovery_target_time = '' # the time stamp up to which recovery will proceed + # (change requires restart) +#recovery_target_xid = '' # the transaction ID up to which recovery will proceed + # (change requires restart) +#recovery_target_lsn = '' # the WAL LSN up to which recovery will proceed + # (change requires restart) +#recovery_target_inclusive = on # Specifies whether to stop: + # just after the specified recovery target (on) + # just before the recovery target (off) + # (change requires restart) +#recovery_target_timeline = 'latest' # 'current', 'latest', or timeline ID + # (change requires restart) +#recovery_target_action = 'pause' # 'pause', 'promote', 'shutdown' + # (change requires restart) + + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +# - Sending Servers - + +# Set these on the primary and on any standby that will send replication data. + +max_wal_senders = 10 # max number of walsender processes + # (change requires restart) +max_replication_slots = 5 # max number of replication slots + # (change requires restart) +#wal_keep_size = 0 # in megabytes; 0 disables +#max_slot_wal_keep_size = -1 # in megabytes; -1 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables +#track_commit_timestamp = off # collect timestamp of transaction commit + # (change requires restart) + +# - Primary Server - + +# These settings are ignored on a standby server. + +#synchronous_standby_names = '' # standby servers that provide sync rep + # method to choose sync standbys, number of sync standbys, + # and comma-separated list of application_name + # from standby(s); '*' = all +#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed + +# - Standby Servers - + +# These settings are ignored on a primary server. + +#primary_conninfo = '' # connection string to sending server +#primary_slot_name = '' # replication slot on sending server +#promote_trigger_file = '' # file name whose presence ends recovery +#hot_standby = on # "off" disallows queries during recovery + # (change requires restart) +#max_standby_archive_delay = 30s # max delay before canceling queries + # when reading WAL from archive; + # -1 allows indefinite delay +#max_standby_streaming_delay = 30s # max delay before canceling queries + # when reading streaming WAL; + # -1 allows indefinite delay +#wal_receiver_create_temp_slot = off # create temp slot if primary_slot_name + # is not set +#wal_receiver_status_interval = 10s # send replies at least this often + # 0 disables +#hot_standby_feedback = off # send info from standby to prevent + # query conflicts +#wal_receiver_timeout = 60s # time that receiver waits for + # communication from primary + # in milliseconds; 0 disables +#wal_retrieve_retry_interval = 5s # time to wait before retrying to + # retrieve WAL after a failed attempt +#recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery + +# - Subscribers - + +# These settings are ignored on a publisher. + +#max_logical_replication_workers = 4 # taken from max_worker_processes + # (change requires restart) +#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers + + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# - Planner Method Configuration - + +#enable_async_append = on +#enable_bitmapscan = on +#enable_gathermerge = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_incremental_sort = on +#enable_indexscan = on +#enable_indexonlyscan = on +#enable_material = on +#enable_resultcache = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_parallel_append = on +#enable_parallel_hash = on +#enable_partition_pruning = on +#enable_partitionwise_join = off +#enable_partitionwise_aggregate = off +#enable_seqscan = on +#enable_sort = on +#enable_tidscan = on + +# - Planner Cost Constants - + +#seq_page_cost = 1.0 # measured on an arbitrary scale +#random_page_cost = 4.0 # same scale as above +#cpu_tuple_cost = 0.01 # same scale as above +#cpu_index_tuple_cost = 0.005 # same scale as above +#cpu_operator_cost = 0.0025 # same scale as above +#parallel_setup_cost = 1000.0 # same scale as above +#parallel_tuple_cost = 0.1 # same scale as above +#min_parallel_table_scan_size = 8MB +#min_parallel_index_scan_size = 512kB +effective_cache_size = 128MB + +#jit_above_cost = 100000 # perform JIT compilation if available + # and query more expensive than this; + # -1 disables +#jit_inline_above_cost = 500000 # inline small functions if query is + # more expensive than this; -1 disables +#jit_optimize_above_cost = 500000 # use expensive JIT optimizations if + # query is more expensive than this; + # -1 disables + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 +#geqo_seed = 0.0 # range 0.0-1.0 + +# - Other Planner Options - + +#default_statistics_target = 100 # range 1-10000 +#constraint_exclusion = partition # on, off, or partition +#cursor_tuple_fraction = 0.1 # range 0.0-1.0 +#from_collapse_limit = 8 +#jit = on # allow JIT compilation +#join_collapse_limit = 8 # 1 disables collapsing of explicit + # JOIN clauses +#plan_cache_mode = auto # auto, force_generic_plan or + # force_custom_plan + + +#------------------------------------------------------------------------------ +# REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +# - Where to Log - + +log_destination = 'csvlog' # Valid values are combinations of + # stderr, csvlog, syslog, and eventlog, + # depending on platform. csvlog + # requires logging_collector to be on. + +# This is used when logging to stderr: +logging_collector = on # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) + +# These are only used if logging_collector is on: +log_directory = 'pg_log' # directory where log files are written, + # can be absolute or relative to PGDATA +log_filename = 'postgresql.log' # log file name pattern, + # can include strftime() escapes +# log_file_mode = 0600 # creation mode for log files, + # begin with 0 to use octal notation +log_rotation_age = 0 # Automatic rotation of logfiles will + # happen after that time. 0 disables. +log_rotation_size = 0 # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. +#log_truncate_on_rotation = off # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' +#syslog_sequence_numbers = on +#syslog_split_messages = on + +# This is only relevant when logging to eventlog (Windows): +# (change requires restart) +#event_source = 'PostgreSQL' + +# - When to Log - + +#log_min_messages = warning # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + +#log_min_error_statement = error # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic (effectively off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements + # and their durations, > 0 logs only + # statements running at least this number + # of milliseconds + +#log_min_duration_sample = -1 # -1 is disabled, 0 logs a sample of statements + # and their durations, > 0 logs only a sample of + # statements running at least this number + # of milliseconds; + # sample fraction is determined by log_statement_sample_rate + +#log_statement_sample_rate = 1.0 # fraction of logged statements exceeding + # log_min_duration_sample to be logged; + # 1.0 logs all such statements, 0.0 never logs + + +#log_transaction_sample_rate = 0.0 # fraction of transactions whose statements + # are logged regardless of their duration; 1.0 logs all + # statements from all transactions, 0.0 never logs + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = on +#log_autovacuum_min_duration = -1 # log autovacuum activity; + # -1 disables, 0 logs all actions and + # their durations, > 0 logs only + # actions running at least this number + # of milliseconds. +#log_checkpoints = off +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_error_verbosity = default # terse, default, or verbose messages +#log_hostname = off +log_line_prefix = '%h %m [%p] %q%u@%d ' # special values: + # %a = application name + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %b = backend type + # %p = process ID + # %P = process ID of parallel group leader + # %t = timestamp without milliseconds + # %m = timestamp with milliseconds + # %n = timestamp with milliseconds (as a Unix epoch) + # %Q = query ID (0 if none or not computed) + # %i = command tag + # %e = SQL state + # %c = session ID + # %l = session line number + # %s = session start timestamp + # %v = virtual transaction ID + # %x = transaction ID (0 if none) + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' +#log_lock_waits = off # log lock waits >= deadlock_timeout +#log_recovery_conflict_waits = off # log standby recovery conflict waits + # >= deadlock_timeout +#log_parameter_max_length = -1 # when logging statements, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_parameter_max_length_on_error = 0 # when logging an error, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_statement = 'none' # none, ddl, mod, all +#log_replication_commands = off +#log_temp_files = -1 # log temporary files equal or larger + # than the specified size in kilobytes; + # -1 disables, 0 logs all temp files +log_timezone = 'UTC' + +#------------------------------------------------------------------------------ +# PROCESS TITLE +#------------------------------------------------------------------------------ + +cluster_name = 'main' # added to process titles if nonempty + # (change requires restart) +#update_process_title = on + + +#------------------------------------------------------------------------------ +# STATISTICS +#------------------------------------------------------------------------------ + +# - Query and Index Statistics Collector - + +#track_activities = on +#track_activity_query_size = 1024 # (change requires restart) +#track_counts = on +#track_io_timing = off +#track_wal_io_timing = off +#track_functions = none # none, pl, all +#stats_temp_directory = 'pg_stat_tmp' + + +# - Monitoring - + +#compute_query_id = auto +#log_statement_stats = off +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off + + +#------------------------------------------------------------------------------ +# AUTOVACUUM +#------------------------------------------------------------------------------ + +#autovacuum = on # Enable autovacuum subprocess? 'on' + # requires track_counts to also be on. +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses + # (change requires restart) +#autovacuum_naptime = 1min # time between autovacuum runs +#autovacuum_vacuum_threshold = 50 # min number of row updates before + # vacuum +#autovacuum_vacuum_insert_threshold = 1000 # min number of row inserts + # before vacuum; -1 disables insert + # vacuums +#autovacuum_analyze_threshold = 50 # min number of row updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum +#autovacuum_vacuum_insert_scale_factor = 0.2 # fraction of inserts over table + # size before insert vacuum +#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum + # (change requires restart) +#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age + # before forced vacuum + # (change requires restart) +#autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for + # autovacuum, in milliseconds; + # -1 means use vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovacuum, -1 means use + # vacuum_cost_limit + + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +# - Statement Behavior - + +#client_min_messages = notice # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error +#search_path = '"$user", public' # schema names +row_security = on +#default_table_access_method = 'heap' +#default_tablespace = '' # a tablespace name, '' uses the default +#default_toast_compression = 'pglz' # 'pglz' or 'lz4' +#temp_tablespaces = '' # a list of tablespace names, '' uses + # only default tablespace +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#default_transaction_deferrable = off +#session_replication_role = 'origin' +#statement_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled +#idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled +#idle_session_timeout = 0 # in milliseconds, 0 is disabled +#vacuum_freeze_table_age = 150000000 +#vacuum_freeze_min_age = 50000000 +#vacuum_failsafe_age = 1600000000 +#vacuum_multixact_freeze_table_age = 150000000 +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_failsafe_age = 1600000000 +#bytea_output = 'hex' # hex, escape +#xmlbinary = 'base64' +#xmloption = 'content' +#gin_pending_list_limit = 4MB + +# - Locale and Formatting - + +#datestyle = 'iso, mdy' +#intervalstyle = 'postgres' +timezone = 'UTC' +#timezone_abbreviations = 'Default' # Select the set of available time zone + # abbreviations. Currently, there are + # Default + # Australia (historical usage) + # India + # You can create your own file in + # share/timezonesets/. +extra_float_digits = 0 # min -15, max 3; any value >0 actually + # selects precise output mode +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# These settings are initialized by initdb, but they can be changed. +lc_messages = 'en_US.UTF-8' # locale for system error message + # strings +lc_monetary = 'en_US.UTF-8' # locale for monetary formatting +lc_numeric = 'en_US.UTF-8' # locale for number formatting +lc_time = 'en_US.UTF-8' # locale for time formatting + +# default configuration for text search +default_text_search_config = 'pg_catalog.english' + +# - Shared Library Preloading - + +#local_preload_libraries = '' +#session_preload_libraries = '' +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron' # (change requires restart) +jit_provider = 'llvmjit' # JIT library to use + +# - Other Defaults - + +#dynamic_library_path = '$libdir' +#gin_fuzzy_search_limit = 0 + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +#deadlock_timeout = 1s +#max_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_relation = -2 # negative values mean + # (max_pred_locks_per_transaction + # / -max_pred_locks_per_relation) - 1 +#max_pred_locks_per_page = 2 # min 0 + + +#------------------------------------------------------------------------------ +# VERSION AND PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# - Previous PostgreSQL Versions - + +#array_nulls = on +#backslash_quote = safe_encoding # on, off, or safe_encoding +#escape_string_warning = on +#lo_compat_privileges = off +#quote_all_identifiers = off +#standard_conforming_strings = on +#synchronize_seqscans = on + +# - Other Platforms and Clients - + +#transform_null_equals = off + + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +#exit_on_error = off # terminate session on any error? +#restart_after_crash = on # reinitialize after backend crash? +#remove_temp_files_after_crash = on # remove temporary files after + # backend crash? +#data_sync_retry = off # retry or panic on failure to fsync + # data? + # (change requires restart) +#recovery_init_sync_method = fsync # fsync, syncfs (Linux 5.8+) + + +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf. Note that these are directives, not variable +# assignments, so they can usefully be given more than once. + +#include_dir = '...' # include files ending in '.conf' from + # a directory, e.g., 'conf.d' +#include_if_exists = '...' # include file only if it exists +#include = '...' # include file + + +#------------------------------------------------------------------------------ +# CUSTOMIZED OPTIONS +#------------------------------------------------------------------------------ + +# Add settings for extensions here \ No newline at end of file diff --git a/ansible/files/postgresql_config/postgresql.service.j2 b/ansible/files/postgresql_config/postgresql.service.j2 new file mode 100644 index 000000000..1a3544bc0 --- /dev/null +++ b/ansible/files/postgresql_config/postgresql.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=PostgreSQL database server +Documentation=man:postgres(1) + +[Service] +Type=notify +User=postgres +ExecStart=/usr/lib/postgresql/bin/postgres -D /etc/postgresql +ExecReload=/bin/kill -HUP $MAINPID +KillMode=mixed +KillSignal=SIGINT +TimeoutSec=0 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/ansible/files/postgrest.service.j2 b/ansible/files/postgrest.service.j2 new file mode 100644 index 000000000..8018a0334 --- /dev/null +++ b/ansible/files/postgrest.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=PostgREST + +[Service] +Type=simple +ExecStart=/opt/postgrest /etc/postgrest.conf +User=postgrest +StandardOutput=file:/var/log/postgrest.stdout +StandardError=file:/var/log/postgrest.error +Slice=services.slice +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/files/queries.yml.j2 b/ansible/files/queries.yml.j2 new file mode 100644 index 000000000..0de4ca2a1 --- /dev/null +++ b/ansible/files/queries.yml.j2 @@ -0,0 +1,194 @@ +pg_database: + query: "SELECT SUM(pg_database_size(pg_database.datname)) / (1024 * 1024) as size_mb FROM pg_database" + master: true + cache_seconds: 30 + metrics: + - size_mb: + usage: "GAUGE" + description: "Disk space used by the database" + +pg_stat_bgwriter: + query: | + select checkpoints_timed as checkpoints_timed_total, + checkpoints_req as checkpoints_req_total, + checkpoint_write_time as checkpoint_write_time_total, + checkpoint_sync_time as checkpoint_sync_time_total, + buffers_checkpoint as buffers_checkpoint_total, + buffers_clean as buffers_clean_total, + maxwritten_clean as maxwritten_clean_total, + buffers_backend as buffers_backend_total, + buffers_backend_fsync as buffers_backend_fsync_total, + buffers_alloc as buffers_alloc_total, + stats_reset + from pg_stat_bgwriter + cache_seconds: 30 + master: true + metrics: + - checkpoints_timed_total: + usage: "COUNTER" + description: "Scheduled checkpoints performed" + - checkpoints_req_total: + usage: "COUNTER" + description: "Requested checkpoints performed" + - checkpoint_write_time_total: + usage: "COUNTER" + description: "Time spent writing checkpoint files to disk" + - checkpoint_sync_time_total: + usage: "COUNTER" + description: "Time spent synchronizing checkpoint files to disk" + - buffers_checkpoint_total: + usage: "COUNTER" + description: "Buffers written during checkpoints" + - buffers_clean_total: + usage: "COUNTER" + description: "Buffers written by bg writter" + - maxwritten_clean_total: + usage: "COUNTER" + description: "Number of times bg writer stopped a cleaning scan because it had written too many buffers" + - buffers_backend_total: + usage: "COUNTER" + description: "Buffers written directly by a backend" + - buffers_backend_fsync_total: + usage: "COUNTER" + description: "fsync calls executed by a backend directly" + - buffers_alloc_total: + usage: "COUNTER" + description: "Buffers allocated" + - stats_reset: + usage: "COUNTER" + description: "Most recent stat reset time" + + +pg_stat_database: + cache_seconds: 30 + query: | + SELECT sum(numbackends) as num_backends, + sum(xact_commit) as xact_commit_total, + sum(xact_rollback) as xact_rollback_total, + sum(blks_read) as blks_read_total, + sum(blks_hit) as blks_hit_total, + sum(tup_returned) as tup_returned_total, + sum(tup_fetched) as tup_fetched_total, + sum(tup_inserted) as tup_inserted_total, + sum(tup_updated) as tup_updated_total, + sum(tup_deleted) as tup_deleted_total, + sum(conflicts) as conflicts_total, + sum(temp_files) as temp_files_total, + sum(temp_bytes) as temp_bytes_total, + sum(deadlocks) as deadlocks_total, + max(stats_reset) as most_recent_reset + FROM pg_stat_database + master: true + metrics: + - num_backends: + usage: "GAUGE" + description: "The number of active backends" + - xact_commit_total: + usage: "COUNTER" + description: "Transactions committed" + - xact_rollback_total: + usage: "COUNTER" + description: "Transactions rolled back" + - blks_read_total: + usage: "COUNTER" + description: "Number of disk blocks read" + - blks_hit_total: + usage: "COUNTER" + description: "Disk blocks found in buffer cache" + - tup_returned_total: + usage: "COUNTER" + description: "Rows returned by queries" + - tup_fetched_total: + usage: "COUNTER" + description: "Rows fetched by queries" + - tup_inserted_total: + usage: "COUNTER" + description: "Rows inserted" + - tup_updated_total: + usage: "COUNTER" + description: "Rows updated" + - tup_deleted_total: + usage: "COUNTER" + description: "Rows deleted" + - conflicts_total: + usage: "COUNTER" + description: "Queries canceled due to conflicts with recovery" + - temp_files_total: + usage: "COUNTER" + description: "Temp files created by queries" + - temp_bytes_total: + usage: "COUNTER" + description: "Temp data written by queries" + - deadlocks_total: + usage: "COUNTER" + description: "Deadlocks detected" + - most_recent_reset: + usage: "COUNTER" + description: "The most recent time one of the databases had its statistics reset" + +pg_stat_database_conflicts: + query: | + SELECT sum(confl_tablespace) as confl_tablespace_total, + sum(confl_lock) as confl_lock_total, + sum(confl_snapshot) as confl_snapshot_total, + sum(confl_bufferpin) as confl_bufferpin_total, + sum(confl_deadlock) as confl_deadlock_total + from pg_stat_database_conflicts + cache_seconds: 30 + master: true + metrics: + - confl_tablespace_total: + usage: "COUNTER" + description: "Queries cancelled due to dropped tablespaces" + - confl_lock_total: + usage: "COUNTER" + description: "Queries cancelled due to lock timeouts" + - confl_snapshot_total: + usage: "COUNTER" + description: "Queries cancelled due to old snapshots" + - confl_bufferpin_total: + usage: "COUNTER" + description: "Queries cancelled due to pinned buffers" + - confl_deadlock_total: + usage: "COUNTER" + description: "Queries cancelled due to deadlocks" + +pg_stat_statements: + query: "SELECT sum(calls) as total_queries, sum(total_exec_time / 1000) as total_time_seconds FROM extensions.pg_stat_statements t1 JOIN pg_database t3 ON (t1.dbid=t3.oid)" + master: true + metrics: + - total_queries: + usage: "COUNTER" + description: "Number of times executed" + - total_time_seconds: + usage: "COUNTER" + description: "Total time spent, in seconds" + +auth_users: + query: "select count(id) as user_count from auth.users" + master: true + cache_seconds: 30 + metrics: + - user_count: + usage: "GAUGE" + description: "Number of users in the project db" + +replication: + query: "SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS realtime_lag_bytes, active AS realtime_slot_status FROM pg_replication_slots where slot_name = 'realtime'" + master: true + metrics: + - realtime_lag_bytes: + usage: "GAUGE" + description: "Replication Lag for Realtime" + - realtime_slot_status: + usage: "GAUGE" + description: "Replication Slot active status" + +storage: + query: "select sum(size) / (1024 * 1024) as storage_size_mb from storage.get_size_by_bucket()" + master: true + cache_seconds: 30 + metrics: + - storage_size_mb: + usage: "GAUGE" + description: "The total size used for all storage buckets, in mb" diff --git a/ansible/files/stat_extension.sql b/ansible/files/stat_extension.sql new file mode 100644 index 000000000..937834004 --- /dev/null +++ b/ansible/files/stat_extension.sql @@ -0,0 +1,2 @@ +CREATE SCHEMA IF NOT exists extensions; +CREATE EXTENSION IF NOT EXISTS pg_stat_statements with schema extensions; diff --git a/ansible/install_roles.yml b/ansible/install_roles.yml deleted file mode 100644 index b8fbce9b1..000000000 --- a/ansible/install_roles.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- - -- name: anxs.postgresql - src: https://github.com/anxs/postgresql - version: v1.12.0 \ No newline at end of file diff --git a/ansible/playbook-docker.yml b/ansible/playbook-docker.yml new file mode 100644 index 000000000..ea699339f --- /dev/null +++ b/ansible/playbook-docker.yml @@ -0,0 +1,52 @@ +- name: Preparing Docker container + hosts: localhost + tasks: + - name: Pull Postgres Image + docker_container: + name: "supabase-postgres-build" + image: "postgres:13.3" + env: + LANGUAGE: "en_US.UTF-8" + LANG: "en_US.UTF-8" + LC_ALL: "en_US.UTF-8" + state: started + memory: 4G + memory_swap: 6G + command: tail -f /dev/null + - name: Add Postgres Image to Ansible Hosts + add_host: + name: "supabase-postgres-build" + ansible_connection: docker + ansible_ssh_user: root + +- name: Build Supabase Postgres + hosts: "supabase-postgres-build" + gather_facts: false + + vars_files: + - ./vars.yml + + tasks: + - name: Setup container + import_tasks: tasks/docker/setup.yml + + - name: Install Postgres extensions + import_tasks: tasks/setup-extensions.yml + + - name: Cleanup container + import_tasks: tasks/docker/cleanup.yml + +- name: Create supabase/postgres docker image + hosts: localhost + tasks: + - name: Commit Docker image + command: docker commit --change='CMD ["postgres"]' "supabase-postgres-build" "supabase/postgres" + +- name: Clean Up Postgres Image + hosts: localhost + tasks: + - name: Remove Running Base Image + docker_container: + name: supabase-postgres-build + state: absent + force_kill: yes \ No newline at end of file diff --git a/ansible/playbook.yml b/ansible/playbook.yml index e38930ec2..8f88921a8 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -3,25 +3,71 @@ pre_tasks: - import_tasks: tasks/setup-system.yml - + vars_files: - ./vars.yml - - roles: - - role: anxs.postgresql + + vars: + sql_files: + - { source: "stat_extension.sql", dest: "01-extension.sql" } + + environment: + PATH: /usr/lib/postgresql/bin:{{ ansible_env.PATH }} tasks: - - name: Install non-Postgres extensions - import_tasks: tasks/setup-misc.yml + - name: Install Postgres from source + import_tasks: tasks/setup-postgres.yml - name: Install Postgres extensions import_tasks: tasks/setup-extensions.yml + - name: Start Postgres Database + systemd: + name: postgresql + state: started + + - name: Install WAL-G + import_tasks: tasks/setup-wal-g.yml + + - name: Install PgBouncer + import_tasks: tasks/setup-pgbouncer.yml + tags: + - install-pgbouncer + + - name: Install PostgREST + import_tasks: tasks/setup-postgrest.yml + tags: + - install-postgrest + + - name: Install Supabase specific content + import_tasks: tasks/setup-supabase-internal.yml + tags: + - install-supabase-internal + - name: Adjust APT update intervals - copy: + copy: src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic + - name: Transfer init SQL files + copy: + src: files/{{ item.source }} + dest: /tmp/{{ item.dest }} + loop: "{{ sql_files }}" + + - name: Execute init SQL files + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/psql -f /tmp/{{ item.dest }} + loop: "{{ sql_files }}" + + - name: Delete SQL scripts + file: + path: /tmp/{{ item.dest }} + state: absent + loop: "{{ sql_files }}" + - name: UFW - Allow SSH connections ufw: rule: allow @@ -30,10 +76,46 @@ - name: UFW - Allow connections to postgreSQL (5432) ufw: rule: allow - port: '5432' + port: "5432" - - name: UFW - Deny all other incoming traffix by default - ufw: + - name: UFW - Allow connections to postgreSQL (6543) + ufw: + rule: allow + port: "6543" + tags: + - install-pgbouncer + + - name: UFW - Deny all other incoming traffic by default + ufw: state: enabled policy: deny - direction: incoming \ No newline at end of file + direction: incoming + + - name: Setup logrotate for postgres logs + copy: + src: files/logrotate-postgres + dest: /etc/logrotate.d/postgres + + - name: Configure logrotation to run every hour + shell: + cmd: mv /etc/cron.daily/logrotate /etc/cron.hourly/ + become: yes + + - name: restart crond + systemd: + state: restarted + name: cron + become: yes + + - name: Enhance fail2ban + import_tasks: tasks/setup-fail2ban.yml + + # Install EC2 instance connect + # Only for AWS images + - name: install EC2 instance connect + become: yes + apt: + pkg: + - ec2-instance-connect + tags: + - aws-only diff --git a/ansible/requirements.yml b/ansible/requirements.yml deleted file mode 100644 index e69de29bb..000000000 diff --git a/ansible/tasks/docker/cleanup.yml b/ansible/tasks/docker/cleanup.yml new file mode 100644 index 000000000..2ccc2af85 --- /dev/null +++ b/ansible/tasks/docker/cleanup.yml @@ -0,0 +1,67 @@ + + +- name: Cleanup - remove build dependencies + apt: + pkg: + - python3 + - rsync + - ca-certificates + - build-essential + - postgresql-server-dev-13 + - curl + - git-core + - gpp + - cpp + - pkg-config + - apt-transport-https + - cmake + - ninja-build + - python + state: absent + +- name: Cleanup - apt update and apt upgrade + apt: update_cache=yes upgrade=yes + # SEE http://archive.vn/DKJjs#parameter-upgrade + +- name: Cleanup - remove dependencies that are no longer required + apt: + autoremove: yes + +- name: Cleanup - remove useless packages from the cache + apt: + autoclean: yes + +- name: Cleanup - reinstall headless jdk + apt: + pkg: + - default-jdk-headless + update_cache: yes + install_recommends: no + +- name: Cleanup - find all files in /tmp + find: + paths: /tmp + file_type: any + register: tmp_items_to_delete + +- name: Cleanup - delete all items in /tmp + file: + path: "/tmp/{{ item.path | basename }}" + state: absent + force: yes + with_items: "{{ tmp_items_to_delete.files }}" + +- name: Cleanup - find all files in /var/lib/apt/lists/* + find: + paths: /var/lib/apt/lists + file_type: any + register: var_items_to_delete + +- name: Cleanup - delete all items in /tmp + file: + path: "/var/lib/apt/lists/{{ item.path | basename }}" + state: absent + force: yes + with_items: "{{ var_items_to_delete.files }}" + + \ No newline at end of file diff --git a/ansible/tasks/docker/setup.yml b/ansible/tasks/docker/setup.yml new file mode 100644 index 000000000..70a54d243 --- /dev/null +++ b/ansible/tasks/docker/setup.yml @@ -0,0 +1,69 @@ +- name: Install Python3 + raw: apt update && apt upgrade -y && apt install python3 -y + +- name: Setup - install common dependencies + apt: + pkg: + - rsync + - ca-certificates + - build-essential + - postgresql-server-dev-13 + - curl + - git-core + - gpp + - cpp + - pkg-config + - apt-transport-https + - cmake + - ninja-build + - python + update_cache: yes + install_recommends: no + +# Find platform architecture and set as a variable +- name: Setup - finding platform architecture + shell: if [ $(uname -m) = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi + register: platform_output +- set_fact: + platform: "{{ platform_output.stdout }}" + +- name: Setup - import postgresql.conf + synchronize: + src: files/postgresql_config/postgresql.conf.j2 + dest: etc/postgresql/postgresql.conf + +- set_fact: + regex_string: "#unix_socket_directories = '/tmp'" + +- name: Setup - modify unix_socket_directories + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: '{{ regex_string }}' + replace: unix_socket_directories = '/var/run/postgresql' + +- name: Setup - modify unix_socket_directories + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: '{{ regex_string }}' + replace: unix_socket_directories = '/var/run/postgresql' + +- name: Setup - modify hba_file directory + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: hba_file = '/etc/postgresql/pg_hba.conf' + replace: hba_file = '/var/lib/postgresql/data/pg_hba.conf' + +- name: Setup - modify ident_file directory + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: ident_file = '/etc/postgresql/pg_ident.conf' + replace: ident_file = '/var/lib/postgresql/data/pg_ident.conf' + +- name: Setup - add init script to /docker-entrypoint-initdb.d + synchronize: + src: files/docker_mnt/init.sh + dest: /docker-entrypoint-initdb.d/init.sh \ No newline at end of file diff --git a/ansible/tasks/internal/node-exporter.yml b/ansible/tasks/internal/node-exporter.yml new file mode 100644 index 000000000..355dcdb15 --- /dev/null +++ b/ansible/tasks/internal/node-exporter.yml @@ -0,0 +1,46 @@ +- name: UFW - Allow connections to node exporter ports + ufw: + rule: allow + port: "9100" + +- name: Node Exporter - download binary archive + get_url: + url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_release }}/node_exporter-{{ node_exporter_release }}.linux-{{ platform }}.tar.gz" + dest: /tmp/node_exporter.tar.gz + checksum: "{{ node_exporter_release_checksum[platform] }}" + +- name: create directories + file: + state: directory + owner: root + path: "/opt/node_exporter" + +- name: node_exporter - unpack archives in /opt + unarchive: + remote_src: yes + src: "/tmp/node_exporter.tar.gz" + dest: /opt/node_exporter + owner: root + extra_opts: [--strip-components=1] + +- name: node_exporter - create service files + template: + src: files/node_exporter.service.j2 + dest: /etc/systemd/system/node_exporter.service + vars: + collectors: + - cpu + - diskstats + - filesystem + - filesystem.ignored-mount-points='^/.+' + - loadavg + - meminfo + - netdev + - netdev.device-exclude='lo' + +- name: node_exporter - reload systemd + systemd: + daemon_reload: yes + enabled: no + state: stopped + name: node_exporter diff --git a/ansible/tasks/internal/optimizations.yml b/ansible/tasks/internal/optimizations.yml new file mode 100644 index 000000000..8ee65518e --- /dev/null +++ b/ansible/tasks/internal/optimizations.yml @@ -0,0 +1,15 @@ +- name: ensure services are stopped + community.general.snap: + name: amazon-ssm-agent + state: absent + +- name: ensure services are stopped and disabled for first boot + systemd: + enabled: no + name: '{{ item }}' + state: stopped + with_items: + - snapd + - postgresql + - pgbouncer + - fail2ban diff --git a/ansible/tasks/internal/postgres-exporter.yml b/ansible/tasks/internal/postgres-exporter.yml new file mode 100644 index 000000000..4fecaeabf --- /dev/null +++ b/ansible/tasks/internal/postgres-exporter.yml @@ -0,0 +1,46 @@ +- name: UFW - Allow connections to exporter for prometheus + ufw: + rule: allow + port: "9187" + +- name: create directories + file: + state: directory + path: "{{ item }}" + owner: root + mode: '0700' + become: yes + with_items: + - /opt/postgres_exporter + - /etc/systemd/system/postgres_exporter.service.d + +- name: download postgres exporter + get_url: + url: "https://github.com/prometheus-community/postgres_exporter/releases/download/v{{ postgres_exporter_release }}/postgres_exporter-{{ postgres_exporter_release }}.linux-{{ platform }}.tar.gz" + dest: /tmp/postgres_exporter.tar.gz + checksum: "{{ postgres_exporter_release_checksum[platform] }}" + +- name: expand postgres exporter + unarchive: + remote_src: yes + src: /tmp/postgres_exporter.tar.gz + dest: /opt/postgres_exporter + extra_opts: [--strip-components=1] + become: yes + +- name: exporter create a service + template: + src: files/postgres_exporter.service.j2 + dest: /etc/systemd/system/postgres_exporter.service + +- name: exporter copy over queries + template: + src: files/queries.yml.j2 + dest: /opt/postgres_exporter/queries.yml + +- name: exporter ensure service is present + systemd: + enabled: no + name: postgres_exporter + daemon_reload: yes + state: stopped diff --git a/ansible/tasks/internal/supautils.yml b/ansible/tasks/internal/supautils.yml new file mode 100644 index 000000000..d6c340971 --- /dev/null +++ b/ansible/tasks/internal/supautils.yml @@ -0,0 +1,45 @@ +# supautils +- name: supautils - download latest release + get_url: + url: "https://github.com/supabase/supautils/archive/refs/tags/v{{ supautils_release }}.tar.gz" + dest: /tmp/supautils-{{ supautils_release }}.tar.gz + checksum: "{{ supautils_release_checksum }}" + +- name: supautils - unpack archive + unarchive: + remote_src: yes + src: /tmp/supautils-{{ supautils_release }}.tar.gz + dest: /tmp + become: yes + +- name: supautils - build + make: + chdir: /tmp/supautils-{{ supautils_release }} + become: yes + +- name: supautils - install + make: + chdir: /tmp/supautils-{{ supautils_release }} + target: install + become: yes + +- name: supautils - set supautils.reserved_roles + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, dashboard_user, pgbouncer, service_role, authenticator, authenticated, anon' + +- name: supautils - set supautils.reserved_memberships + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program' + +- name: supautils - add supautils to shared_preload_libraries + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: shared_preload_libraries = ' + replace: shared_preload_libraries = 'supautils, diff --git a/ansible/tasks/postgres-extensions/01-postgis.yml b/ansible/tasks/postgres-extensions/01-postgis.yml new file mode 100644 index 000000000..9aaad6d5e --- /dev/null +++ b/ansible/tasks/postgres-extensions/01-postgis.yml @@ -0,0 +1,77 @@ +# postgis +- name: postgis - download & install dependencies + apt: + pkg: + - libgeos-dev + - libproj-dev + - libgdal-dev + - libjson-c-dev + - libxml2-dev + - libboost-all-dev + - libcgal-dev + - libmpfr-dev + - libgmp-dev + - cmake + update_cache: yes + cache_valid_time: 3600 + install_recommends: no + +- name: postgis - download SFCGAL dependency + get_url: + url: "https://gitlab.com/Oslandia/SFCGAL/-/archive/v{{ sfcgal_release }}/SFCGAL-v{{ sfcgal_release }}.tar.gz" + dest: /tmp/SFCGAL-v{{ sfcgal_release }}.tar.gz + checksum: "{{ sfcgal_release_checksum }}" + +- name: postgis - unpack SFCGAL + unarchive: + remote_src: yes + src: /tmp/SFCGAL-v{{ sfcgal_release }}.tar.gz + dest: /tmp + become: yes + +- name: postgis - compile SFCGAL + shell: + cmd: "cmake ." + chdir: /tmp/SFCGAL-v{{ sfcgal_release }} + become: yes + +- name: postgis - build SFCGAL + make: + chdir: /tmp/SFCGAL-v{{ sfcgal_release }} + become: yes + +- name: postgis - install SFCGAL + make: + chdir: /tmp/SFCGAL-v{{ sfcgal_release }} + target: install + become: yes + +- name: postgis - download latest release + get_url: + url: "https://download.osgeo.org/postgis/source/postgis-{{ postgis_release }}.tar.gz" + dest: /tmp/postgis-{{ postgis_release }}.tar.gz + checksum: "{{ postgis_release_checksum }}" + +- name: postgis - unpack archive + unarchive: + remote_src: yes + src: /tmp/postgis-{{ postgis_release }}.tar.gz + dest: /tmp + become: yes + +- name: postgis - configure + shell: + cmd: "./configure --without-protobuf --with-sfcgal" + chdir: /tmp/postgis-{{ postgis_release }} + become: yes + +- name: postgis - build + make: + chdir: /tmp/postgis-{{ postgis_release }} + become: yes + +- name: postgis - install + make: + chdir: /tmp/postgis-{{ postgis_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/02-pgrouting.yml b/ansible/tasks/postgres-extensions/02-pgrouting.yml new file mode 100644 index 000000000..9020d0024 --- /dev/null +++ b/ansible/tasks/postgres-extensions/02-pgrouting.yml @@ -0,0 +1,36 @@ +# pgRouting +- name: pgRouting - download latest release + get_url: + url: "https://github.com/pgRouting/pgrouting/releases/download/v{{ pgrouting_release }}/pgrouting-{{ pgrouting_release }}.tar.gz" + dest: /tmp/pgrouting-{{ pgrouting_release }}.tar.gz + checksum: "{{ pgrouting_release_checksum }}" + +- name: pgRouting - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgrouting-{{ pgrouting_release }}.tar.gz + dest: /tmp + become: yes + +- name: pgRouting - create build directory + file: + path: /tmp/pgrouting-{{ pgrouting_release }}/build + state: directory + become: yes + +- name: pgRouting - compile + shell: + cmd: "cmake -DBUILD_HTML=OFF -DBUILD_DOXY=OFF .." + chdir: /tmp/pgrouting-{{ pgrouting_release }}/build + become: yes + +- name: pgRouting - build + make: + chdir: /tmp/pgrouting-{{ pgrouting_release }}/build + become: yes + +- name: pgRouting - install + make: + chdir: /tmp/pgrouting-{{ pgrouting_release }}/build + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/03-pgtap.yml b/ansible/tasks/postgres-extensions/03-pgtap.yml new file mode 100644 index 000000000..6dc11f0e3 --- /dev/null +++ b/ansible/tasks/postgres-extensions/03-pgtap.yml @@ -0,0 +1,19 @@ +# pgTAP +- name: pgTAP - download latest release + get_url: + url: "https://github.com/theory/pgtap/archive/v{{ pgtap_release }}.tar.gz" + dest: /tmp/pgtap-{{ pgtap_release }}.tar.gz + checksum: "{{ pgtap_release_checksum }}" + +- name: pgTAP - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgtap-{{ pgtap_release }}.tar.gz + dest: /tmp + become: yes + +- name: pgTAP - install + make: + chdir: /tmp/pgtap-{{ pgtap_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/04-pg_cron.yml b/ansible/tasks/postgres-extensions/04-pg_cron.yml new file mode 100644 index 000000000..5bdf294f5 --- /dev/null +++ b/ansible/tasks/postgres-extensions/04-pg_cron.yml @@ -0,0 +1,31 @@ +# pg_cron +- name: pg_cron - download latest release + get_url: + url: "https://github.com/citusdata/pg_cron/archive/refs/tags/v{{ pg_cron_release }}.tar.gz" + dest: /tmp/pg_cron-{{ pg_cron_release }}.tar.gz + checksum: "{{ pg_cron_release_checksum }}" + +- name: pg_cron - unpack archive + unarchive: + remote_src: yes + src: /tmp/pg_cron-{{ pg_cron_release }}.tar.gz + dest: /tmp + become: yes + +- name: pg_cron - build + make: + chdir: /tmp/pg_cron-{{ pg_cron_release }} + become: yes + +- name: pg_cron - install + make: + chdir: /tmp/pg_cron-{{ pg_cron_release }} + target: install + become: yes + +- name: pg_cron - set cron.database_name + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: cron.database_name = 'postgres' \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/05-pgaudit.yml b/ansible/tasks/postgres-extensions/05-pgaudit.yml new file mode 100644 index 000000000..6d3b2bca7 --- /dev/null +++ b/ansible/tasks/postgres-extensions/05-pgaudit.yml @@ -0,0 +1,37 @@ +# pgAudit +- name: pgAudit - download & install dependencies + apt: + pkg: + - libssl-dev + - libkrb5-dev + update_cache: yes + install_recommends: no + +- name: pgAudit - download latest release + get_url: + url: "https://github.com/pgaudit/pgaudit/archive/refs/tags/{{ pgaudit_release }}.tar.gz" + dest: /tmp/pgaudit-{{ pgaudit_release }}.tar.gz + checksum: "{{ pgaudit_release_checksum }}" + +- name: pgAudit - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgaudit-{{ pgaudit_release }}.tar.gz + dest: /tmp + become: yes + +- name: pgAudit - build + make: + chdir: /tmp/pgaudit-{{ pgaudit_release }} + target: check + params: + USE_PGXS: 1 + become: yes + +- name: pgAudit - install + make: + chdir: /tmp/pgaudit-{{ pgaudit_release }} + target: install + params: + USE_PGXS: 1 + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/06-pgjwt.yml b/ansible/tasks/postgres-extensions/06-pgjwt.yml new file mode 100644 index 000000000..b2734e130 --- /dev/null +++ b/ansible/tasks/postgres-extensions/06-pgjwt.yml @@ -0,0 +1,12 @@ +# pgjwt +- name: pgjwt - download from master branch + git: + repo: https://github.com/michelp/pgjwt.git + dest: /tmp/pgjwt + version: master + +- name: pgjwt - install + make: + chdir: /tmp/pgjwt + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/07-pgsql-http.yml b/ansible/tasks/postgres-extensions/07-pgsql-http.yml new file mode 100644 index 000000000..6fd5cf9aa --- /dev/null +++ b/ansible/tasks/postgres-extensions/07-pgsql-http.yml @@ -0,0 +1,37 @@ +# pgsql-http +- name: pgsql-http - libcurl4 package + apt: + pkg: + - libcurl4 + state: absent + +- name: pgsql-http - download & install dependencies + apt: + pkg: + - libcurl4-gnutls-dev + update_cache: yes + install_recommends: no + +- name: pgsql-http - download latest release + get_url: + url: "https://github.com/pramsey/pgsql-http/archive/refs/tags/v{{ pgsql_http_release }}.tar.gz" + dest: /tmp/pgsql_http-{{ pgsql_http_release }}.tar.gz + checksum: "{{ pgsql_http_release_checksum }}" + +- name: pgsql-http - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgsql_http-{{ pgsql_http_release }}.tar.gz + dest: /tmp + become: yes + +- name: pgsql-http - build + make: + chdir: /tmp/pgsql-http-{{ pgsql_http_release }} + become: yes + +- name: pgsql-http - install + make: + chdir: /tmp/pgsql-http-{{ pgsql_http_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/08-plpgsql_check.yml b/ansible/tasks/postgres-extensions/08-plpgsql_check.yml new file mode 100644 index 000000000..16fb5aa7e --- /dev/null +++ b/ansible/tasks/postgres-extensions/08-plpgsql_check.yml @@ -0,0 +1,32 @@ +# plpgsql_check +- name: plpgsql_check - download & install dependencies + apt: + pkg: + - libicu-dev + update_cache: yes + install_recommends: no + +- name: plpgsql_check - download latest release + get_url: + url: "https://github.com/okbob/plpgsql_check/archive/refs/tags/v{{ plpgsql_check_release }}.tar.gz" + dest: /tmp/plpgsql_check-{{ plpgsql_check_release }}.tar.gz + checksum: "{{ plpgsql_check_release_checksum }}" + +- name: plpgsql_check - unpack archive + unarchive: + remote_src: yes + src: /tmp/plpgsql_check-{{ plpgsql_check_release }}.tar.gz + dest: /tmp + become: yes + +- name: plpgsql_check - clean + make: + chdir: /tmp/plpgsql_check-{{ plpgsql_check_release }} + target: clean + become: yes + +- name: plpgsql_check - install + make: + chdir: /tmp/plpgsql_check-{{ plpgsql_check_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/09-pg-safeupdate.yml b/ansible/tasks/postgres-extensions/09-pg-safeupdate.yml new file mode 100644 index 000000000..e27cfd9dc --- /dev/null +++ b/ansible/tasks/postgres-extensions/09-pg-safeupdate.yml @@ -0,0 +1,24 @@ +# pg-safeupdate +- name: pg-safeupdate - download latest release + get_url: + url: "https://github.com/eradman/pg-safeupdate/archive/refs/tags/{{ pg_safeupdate_release }}.tar.gz" + dest: /tmp/pg_safeupdate-{{ pg_safeupdate_release }}.tar.gz + checksum: "{{ pg_safeupdate_release_checksum }}" + +- name: pg-safeupdate - unpack archive + unarchive: + remote_src: yes + src: /tmp/pg_safeupdate-{{ pg_safeupdate_release }}.tar.gz + dest: /tmp + become: yes + +- name: pg-safeupdate - build + make: + chdir: /tmp/pg-safeupdate-{{ pg_safeupdate_release }} + become: yes + +- name: pg-safeupdate - install + make: + chdir: /tmp/pg-safeupdate-{{ pg_safeupdate_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/10-timescaledb.yml b/ansible/tasks/postgres-extensions/10-timescaledb.yml new file mode 100644 index 000000000..46798998b --- /dev/null +++ b/ansible/tasks/postgres-extensions/10-timescaledb.yml @@ -0,0 +1,31 @@ +# timescaledb +- name: timescaledb - download & install dependencies + apt: + pkg: + - cmake + update_cache: yes + install_recommends: no + +- name: timescaledb - download latest release + git: + repo: https://github.com/timescale/timescaledb.git + dest: /tmp/timescaledb + version: "{{ timescaledb_release }}" + become: yes + +- name: timescaledb - bootstrap + shell: + cmd: "./bootstrap -DAPACHE_ONLY=1 -DREGRESS_CHECKS=OFF" + chdir: /tmp/timescaledb + become: yes + +- name: timescaledb - build + make: + chdir: /tmp/timescaledb/build + become: yes + +- name: timescaledb - install + make: + chdir: /tmp/timescaledb/build + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/11-wal2json.yml b/ansible/tasks/postgres-extensions/11-wal2json.yml new file mode 100644 index 000000000..8fabbdd09 --- /dev/null +++ b/ansible/tasks/postgres-extensions/11-wal2json.yml @@ -0,0 +1,24 @@ +# wal2json +- name: wal2json - download latest release + get_url: + url: "https://github.com/eulerto/wal2json/archive/refs/tags/wal2json_{{ wal2json_release }}.tar.gz" + dest: /tmp/wal2json-{{ wal2json_release }}.tar.gz + checksum: "{{ wal2json_release_checksum }}" + +- name: wal2json - unpack archive + unarchive: + remote_src: yes + src: /tmp/wal2json-{{ wal2json_release }}.tar.gz + dest: /tmp + become: yes + +- name: wal2json - build + make: + chdir: /tmp/wal2json-wal2json_{{ wal2json_release }} + become: yes + +- name: wal2json - install + make: + chdir: /tmp/wal2json-wal2json_{{ wal2json_release }} + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/12-pljava.yml b/ansible/tasks/postgres-extensions/12-pljava.yml new file mode 100644 index 000000000..3bea59cb9 --- /dev/null +++ b/ansible/tasks/postgres-extensions/12-pljava.yml @@ -0,0 +1,62 @@ +# pljava +- name: pljava - download & install dependencies + apt: + pkg: + - maven + - default-jre + - default-jdk + update_cache: yes + install_recommends: no + +- name: pljava - download latest release + get_url: + url: https://github.com/tada/pljava/archive/V{{ pljava_release }}.tar.gz + dest: /tmp/pljava-{{ pljava_release }}.tar.gz + checksum: "{{ pljava_release_checksum }}" + +- name: pljava - unpack archive + unarchive: + remote_src: yes + src: /tmp/pljava-{{ pljava_release }}.tar.gz + dest: /tmp + become: yes + +- name: pljava - build + become: yes + shell: + cmd: mvn clean install + chdir: /tmp/pljava-{{ pljava_release }} + +- name: pljava - install + become: yes + shell: + cmd: java -jar pljava-packaging/target/pljava-pg13.jar + chdir: /tmp/pljava-{{ pljava_release }} + +- name: pljava - remove build dependencies + apt: + pkg: + - maven + - default-jre + - default-jdk + state: absent + +- name: pljava - install headless jdk + apt: + pkg: + - default-jdk-headless + update_cache: yes + install_recommends: no + +- name: pljava - set pljava.libjvm_location + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-{{ platform }}/lib/server/libjvm.so' + +- name: pljava - remove ~/.m2 directory + become: yes + file: + path: ~/.m2 + state: absent \ No newline at end of file diff --git a/ansible/tasks/postgres-extensions/13-plv8.yml b/ansible/tasks/postgres-extensions/13-plv8.yml new file mode 100644 index 000000000..1966649ca --- /dev/null +++ b/ansible/tasks/postgres-extensions/13-plv8.yml @@ -0,0 +1,48 @@ +# plv8 +- name: plv8 - download & install dependencies + apt: + pkg: + - build-essential + - ca-certificates + - curl + - git-core + - gpp + - cpp + - pkg-config + - apt-transport-https + - cmake + - libc++-dev + - libc++abi-dev + - libc++1 + - libglib2.0-dev + - libtinfo5 + - libc++abi1 + - ninja-build + - python + update_cache: yes + install_recommends: no + +- name: plv8 - download latest release + git: + repo: https://github.com/plv8/plv8.git + dest: /tmp/plv8 + version: 3656177d384e3e02b74faa8e2931600f3690ab59 + become: yes + +- name: Create a symbolic link + file: + src: /lib/aarch64-linux-gnu/libc++.so.1 + dest: /lib/aarch64-linux-gnu/libc++.so + state: link + when: platform == "arm64" + +- name: plv8 - build + make: + chdir: /tmp/plv8 + become: yes + +- name: plv8 - install + make: + chdir: /tmp/plv8 + target: install + become: yes \ No newline at end of file diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index 85cd46253..a9f2d3f23 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -1,271 +1,38 @@ -# postgis -- name: Install postgis for postgreSQL versions < 10 - apt: - pkg: - - libgeos-c1v5 - - "postgresql-{{ postgresql_version }}-postgis-{{ postgresql_ext_postgis_version }}" - - "postgresql-{{ postgresql_version }}-postgis-scripts" - update_cache: yes - cache_valid_time: 3600 - when: postgresql_version < 10 +- name: Install postgis + import_tasks: tasks/postgres-extensions/01-postgis.yml -- name: Install postgis for postgreSQL versions >= 10 - apt: - pkg: - - libgeos-c1v5 - - "postgresql-{{ postgresql_version }}-postgis-{{ postgresql_ext_postgis_version }}" - - "postgresql-{{ postgresql_version }}-postgis-{{ postgresql_ext_postgis_version }}-scripts" - update_cache: yes - cache_valid_time: 3600 - when: postgresql_version >= 10 +- name: Install pgrouting + import_tasks: tasks/postgres-extensions/02-pgrouting.yml -# pgTAP -- name: pgTAP - download latest release - get_url: - url: "https://github.com/theory/pgtap/archive/v{{ pgtap_release }}.tar.gz" - dest: /tmp - checksum: "{{ pgtap_release_checksum }}" +- name: Install pgtap + import_tasks: tasks/postgres-extensions/03-pgtap.yml -- name: pgTAP - unpack archive - unarchive: - remote_src: yes - src: /tmp/pgtap-{{ pgtap_release }}.tar.gz - dest: /tmp - become: yes +- name: Install pg_cron + import_tasks: tasks/postgres-extensions/04-pg_cron.yml -- name: pgTAP - install - make: - chdir: /tmp/pgtap-{{ pgtap_release }} - target: install - become: yes +- name: Install pgaudit + import_tasks: tasks/postgres-extensions/05-pgaudit.yml -# plpython -- name: Install plpython - apt: - pkg: postgresql-plpython3-12 - update_cache: yes - cache_valid_time: 3600 +- name: Install pgjwt + import_tasks: tasks/postgres-extensions/06-pgjwt.yml -# pgAudit -- name: pgAudit - download & install dependencies - apt: - pkg: - - postgresql-server-dev-12 - - libssl-dev - - libkrb5-dev - update_cache: yes - install_recommends: no +- name: Install pgsql-http + import_tasks: tasks/postgres-extensions/07-pgsql-http.yml -- name: pgAudit - download latest release - get_url: - url: "https://github.com/pgaudit/pgaudit/archive/{{ pgaudit_release }}.tar.gz" - dest: /tmp - checksum: "{{ pgaudit_release_checksum }}" +- name: Install plpgsql_check + import_tasks: tasks/postgres-extensions/08-plpgsql_check.yml -- name: pgAudit - unpack archive - unarchive: - remote_src: yes - src: /tmp/pgaudit-{{ pgaudit_release }}.tar.gz - dest: /tmp - become: yes +- name: Install pg-safeupdate + import_tasks: tasks/postgres-extensions/09-pg-safeupdate.yml -- name: pgAudit - build - make: - chdir: /tmp/pgaudit-{{ pgaudit_release }} - target: check - params: - USE_PGXS: 1 - become: yes +# - name: Install timescaledb +# import_tasks: tasks/postgres-extensions/10-timescaledb.yml -- name: pgAudit - install - make: - chdir: /tmp/pgaudit-{{ pgaudit_release }} - target: install - params: - USE_PGXS: 1 - become: yes +- name: Install wal2json + import_tasks: tasks/postgres-extensions/11-wal2json.yml -# pgjwt -- name: pgjwt - download from master branch - git: - repo: https://github.com/michelp/pgjwt.git - dest: /tmp/pgjwt - version: master +- name: Install pljava + import_tasks: tasks/postgres-extensions/12-pljava.yml -- name: pgjwt - install - make: - chdir: /tmp/pgjwt - target: install - become: yes - -- name: Remove libcurl4 package - apt: - pkg: - - libcurl4 - state: absent - -# pgsql-http -- name: pgsql-http - download & install dependencies - apt: - pkg: - - libcurl4-gnutls-dev - update_cache: yes - install_recommends: yes - -- name: pgsql-http - download latest release - get_url: - url: "https://github.com/pramsey/pgsql-http/archive/v{{ pgsql_http_release }}.tar.gz" - dest: /tmp - checksum: "{{ pgsql_http_release_checksum }}" - -- name: pgsql-http - unpack archive - unarchive: - remote_src: yes - src: /tmp/pgsql-http-{{ pgsql_http_release }}.tar.gz - dest: /tmp - become: yes - -- name: pgsql-http - build - make: - chdir: /tmp/pgsql-http-{{ pgsql_http_release }} - become: yes - -- name: pgsql-http - install - make: - chdir: /tmp/pgsql-http-{{ pgsql_http_release }} - target: install - become: yes - -# plpgsql_check -- name: plpgsql_check - download & install dependencies - apt: - pkg: - - libicu-dev - update_cache: yes - install_recommends: no - -- name: plpgsql_check - download latest release - get_url: - url: https://github.com/okbob/plpgsql_check/archive/v{{ plpgsql_check_release }}.tar.gz - dest: /tmp - checksum: "{{ plpgsql_check_release_checksum }}" - -- name: plpgsql_check - unpack archive - unarchive: - remote_src: yes - src: /tmp/plpgsql_check-{{ plpgsql_check_release }}.tar.gz - dest: /tmp - become: yes - -- name: plpgsql_check - clean - make: - chdir: /tmp/plpgsql_check-{{ plpgsql_check_release }} - target: clean - become: yes - -- name: plpgsql_check - install - make: - chdir: /tmp/plpgsql_check-{{ plpgsql_check_release }} - target: install - become: yes - -# pljava -- name: pljava - download & install dependencies - apt: - pkg: - - maven - - default-jre - - default-jdk - update_cache: yes - install_recommends: yes - -- name: pljava - download latest release - get_url: - url: https://github.com/tada/pljava/archive/V{{ pljava_release }}.tar.gz - dest: /tmp - checksum: "{{ pljava_release_checksum }}" - -- name: pljava - unpack archive - unarchive: - remote_src: yes - src: /tmp/pljava-{{ pljava_release }}.tar.gz - dest: /tmp - become: yes - -- name: pljava - build - become: yes - shell: - cmd: mvn clean install - chdir: /tmp/pljava-{{ pljava_release }} - -- name: pljava - install - become: yes - shell: - cmd: java -jar pljava-packaging/target/pljava-pg12.3-amd64-Linux-gpp.jar - chdir: /tmp/pljava-{{ pljava_release }} - -- name: pljava - remove build dependencies - apt: - pkg: - - maven - - default-jre - - default-jdk - state: absent - -- name: pljava - install headless jdk - apt: - pkg: - - default-jdk-headless - update_cache: yes - install_recommends: no - -- name: pljava - set pljava.libjvm_location - become: yes - shell: - cmd: echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-amd64/lib/server/libjvm.so'" >> /etc/postgresql/12/main/postgresql.conf - -- name: pljava - remove ~/.m2 directory - become: yes - file: - path: ~/.m2 - state: absent - -# plv8 -- name: plv8 - download & install dependencies - apt: - pkg: - - build-essential - - ca-certificates - - curl - - git-core - - gpp - - cpp - - pkg-config - - apt-transport-https - - cmake - - libc++-dev - - libc++abi-dev - - postgresql-server-dev-12 - - libc++1 - - libtinfo5 - - libc++abi1 - update_cache: yes - install_recommends: no - -- name: plv8 - download latest release - git: - repo: https://github.com/plv8/plv8.git - dest: /tmp/plv8 - version: r3.0alpha - become: yes - -- name: plv8 - build - make: - chdir: /tmp/plv8 - become: yes - -- name: plv8 - install - make: - chdir: /tmp/plv8 - target: install - become: yes +- name: Install plv8 + import_tasks: tasks/postgres-extensions/13-plv8.yml \ No newline at end of file diff --git a/ansible/tasks/setup-fail2ban.yml b/ansible/tasks/setup-fail2ban.yml new file mode 100644 index 000000000..7b0666b07 --- /dev/null +++ b/ansible/tasks/setup-fail2ban.yml @@ -0,0 +1,41 @@ +# set default bantime to 30 minutes +- name: supautils - add supautils to shared_preload_libraries + become: yes + replace: + path: /etc/fail2ban/jail.conf + regexp: bantime = 10m + replace: bantime = 1800 + +# postgresql +- name: import jail.d/postgresql.conf + template: + src: files/fail2ban_config/jail-postgresql.conf.j2 + dest: /etc/fail2ban/jail.d/postgresql.conf + become: yes + +- name: import filter.d/postgresql.conf + template: + src: files/fail2ban_config/filter-postgresql.conf.j2 + dest: /etc/fail2ban/filter.d/postgresql.conf + become: yes + +- name: add in supabase specific ignore filters + lineinfile: + path: /etc/fail2ban/filter.d/postgresql.conf + state: present + line: "{{ item.line }}" + loop: + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_admin".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_auth_admin".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_storage_admin".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""authenticator".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""pgbouncer".*$' } + become: yes + tags: + - install-supabase-internal + +# Restart +- name: fail2ban - restart + systemd: + name: fail2ban + state: restarted \ No newline at end of file diff --git a/ansible/tasks/setup-misc.yml b/ansible/tasks/setup-misc.yml deleted file mode 100644 index e67d6a364..000000000 --- a/ansible/tasks/setup-misc.yml +++ /dev/null @@ -1,24 +0,0 @@ -# WAL-G -- name: Install daemontools - become: yes - apt: - pkg: - - daemontools - -- name: WAL-G - download latest release - get_url: - url: https://github.com/wal-g/wal-g/releases/download/v{{ wal_g_release }}/wal-g.linux-amd64.tar.gz - dest: /tmp - checksum: "{{ wal_g_release_checksum }}" - -- name: WAL-G - unpack archive - unarchive: - remote_src: yes - src: /tmp/wal-g.linux-amd64.tar.gz - dest: /tmp - become: yes - -- name: WAL-G - install - become: yes - shell: - cmd: mv /tmp/wal-g /usr/local/bin/ diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml new file mode 100644 index 000000000..7400b19d9 --- /dev/null +++ b/ansible/tasks/setup-pgbouncer.yml @@ -0,0 +1,111 @@ +# PgBouncer +- name: PgBouncer - download & install dependencies + apt: + pkg: + - libssl-dev + - pkg-config + - libevent-dev + update_cache: yes + cache_valid_time: 3600 + +- name: PgBouncer - download latest release + get_url: + url: "https://www.pgbouncer.org/downloads/files/{{ pgbouncer_release }}/pgbouncer-{{ pgbouncer_release }}.tar.gz" + dest: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz + checksum: "{{ pgbouncer_release_checksum }}" + +- name: PgBouncer - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz + dest: /tmp + become: yes + +- name: PgBouncer - configure + shell: + cmd: "./configure --prefix=/usr/local --with-systemd" + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + become: yes + +- name: PgBouncer - build + make: + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + become: yes + +- name: PgBouncer - install + make: + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + target: install + become: yes + +# Create /etc/postgresql directory and make sure postgres group owns it +- name: PgBouncer - create a directory if it does not exist + file: + path: /etc/pgbouncer + state: directory + group: postgres + +- name: PgBouncer - adjust pgbouncer.ini + copy: + src: files/pgbouncer_config/pgbouncer.ini.j2 + dest: /etc/pgbouncer/pgbouncer.ini + +- name: PgBouncer - create a directory if it does not exist + file: + path: /etc/pgbouncer/userlist.txt + state: touch + group: postgres + owner: postgres + +- name: import /etc/tmpfiles.d/pgbouncer.conf + template: + src: files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 + dest: /etc/tmpfiles.d/pgbouncer.conf + become: yes + +- name: PgBouncer - add permissions for pgbouncer user + become: yes + lineinfile: + path: /etc/postgresql/pg_hba.conf + state: present + insertafter: '# Default:' + line: "{{ item }}" + with_items: + - "host all pgbouncer 127.0.0.1/32 md5" + - "# Allow connection by pgbouncer user" + +# Run PgBouncer SQL script +- name: Transfer init SQL files + copy: + src: files/pgbouncer_config/pgbouncer_auth_schema.sql + dest: /tmp/00-schema.sql + +- name: Execute init SQL files + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/psql -f /tmp/00-schema.sql + +# Add fail2ban filter +- name: import jail.d/pgbouncer.conf + template: + src: files/fail2ban_config/jail-pgbouncer.conf.j2 + dest: /etc/fail2ban/jail.d/pgbouncer.conf + become: yes + +- name: import filter.d/pgbouncer.conf + template: + src: files/fail2ban_config/filter-pgbouncer.conf.j2 + dest: /etc/fail2ban/filter.d/pgbouncer.conf + become: yes + +# Add systemd file for PgBouncer +- name: PgBouncer - import postgresql.service + template: + src: files/pgbouncer_config/pgbouncer.service.j2 + dest: /etc/systemd/system/pgbouncer.service + become: yes + +- name: PgBouncer - reload systemd + systemd: + daemon_reload: yes \ No newline at end of file diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml new file mode 100644 index 000000000..679c3ceb4 --- /dev/null +++ b/ansible/tasks/setup-postgres.yml @@ -0,0 +1,165 @@ +# Downloading dependencies +- name: Postgres dependencies + become: yes + apt: + pkg: + - build-essential + - libreadline-dev + - zlib1g-dev + - flex + - bison + - libxml2-dev + - libxslt-dev + - libssl-dev + - libsystemd-dev + - libpq-dev + - libxml2-utils + - uuid-dev + - xsltproc + - ssl-cert + +- name: Download LLVM & Clang + become: yes + apt: + pkg: + - llvm-11-dev + - clang-11 + +- name: Download GCC 10 + become: yes + apt: + pkg: + - gcc-10 + - g++-10 + +- name: Switch to GCC 10 + shell: + cmd: update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 100 --slave /usr/bin/g++ g++ /usr/bin/g++-10 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 + become: yes + +# Building Postgres from source +- name: Postgres - download latest release + get_url: + url: https://ftp.postgresql.org/pub/source/v{{ postgresql_release }}/postgresql-{{ postgresql_release }}.tar.gz + dest: /tmp + checksum: "{{ postgresql_release_checksum }}" + +- name: Postgres - unpack archive + unarchive: + remote_src: yes + src: /tmp/postgresql-{{ postgresql_release }}.tar.gz + dest: /tmp + become: yes + +- name: Setting CFLAGS (arm) + set_fact: + cflags: "-moutline-atomics -mtune=neoverse-n1 -fsigned-char" + when: platform == "arm64" + +- name: Setting CFLAGS (x86) + set_fact: + cflags: "-fsigned-char" + when: platform == "amd64" + +- name: Postgres - configure + shell: + cmd: CFLAGS='{{ cflags }}' LLVM_CONFIG=/usr/bin/llvm-config-11 CLANG=/usr/bin/clang-11 ./configure --with-llvm --with-openssl --with-systemd --with-uuid=e2fs --exec-prefix=/usr/lib/postgresql --datarootdir=/var/lib/postgresql + chdir: /tmp/postgresql-{{ postgresql_release }} + become: yes + +- name: Postgres - build + make: + target: world + chdir: /tmp/postgresql-{{ postgresql_release }} + become: yes + +- name: Postgres - install + make: + target: install-world + chdir: /tmp/postgresql-{{ postgresql_release }} + become: yes + +# Create postgres user +- name: Create postgres user + user: + name: postgres + shell: /bin/false + comment: Postgres user + groups: ssl-cert + +- name: Recursively change ownership of a directory + file: + path: /var/lib/postgresql + state: directory + recurse: yes + owner: postgres + group: postgres + +# Create /etc/postgresql directory and make sure postgres group owns it +- name: Create a directory if it does not exist + file: + path: /etc/postgresql + state: directory + owner: postgres + group: postgres + +# Move Postgres configuration files into /etc/postgresql +# Add postgresql.conf +- name: import postgresql.conf + template: + src: files/postgresql_config/postgresql.conf.j2 + dest: /etc/postgresql/postgresql.conf + group: postgres + +# Add pg_hba.conf +- name: import pg_hba.conf + template: + src: files/postgresql_config/pg_hba.conf.j2 + dest: /etc/postgresql/pg_hba.conf + group: postgres + +# Add pg_ident.conf +- name: import pg_ident.conf + template: + src: files/postgresql_config/pg_ident.conf.j2 + dest: /etc/postgresql/pg_ident.conf + group: postgres + +- name: Find all files in /usr/lib/postgresql/bin + find: + paths: /usr/lib/postgresql/bin + register: postgresql_bin + +- name: Create symbolic links for Postgres binaries to /usr/bin/ + become: True + file: + src: "{{ item.path }}" + path: "/usr/bin/{{ item.path | basename }}" + state: link + force: yes + with_items: "{{ postgresql_bin.files }}" + +# init DB +- name: Initialize the database + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb + vars: + ansible_command_timeout: 60 + # Circumvents the following error: + # "Timeout (12s) waiting for privilege escalation prompt" + +# Add systemd file for Postgres +- name: import postgresql.service + template: + src: files/postgresql_config/postgresql.service.j2 + dest: /etc/systemd/system/postgresql.service + become: yes + +# Reload +- name: System - systemd reload + systemd: + enabled: yes + name: postgresql + daemon_reload: yes diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml new file mode 100644 index 000000000..16a10e8c7 --- /dev/null +++ b/ansible/tasks/setup-postgrest.yml @@ -0,0 +1,39 @@ +- name: PostgREST - system user + user: name=postgrest + +# libpq is a C library that enables user programs to communicate with +# the PostgreSQL database server. +- name: PostgREST - system dependencies + apt: + pkg: + - libpq5 + +- name: PostgREST - download ubuntu binary archive (arm) + get_url: + url: "https://github.com/PostgREST/postgrest/releases/download/nightly/postgrest-nightly-{{ postgrest_arm_release }}.tar.xz" + dest: /tmp/postgrest.tar.xz + checksum: "{{ postgrest_arm_release_checksum }}" + when: platform == "arm64" + +- name: PostgREST - download ubuntu binary archive (x86) + get_url: + url: "https://github.com/PostgREST/postgrest/releases/download/nightly/postgrest-nightly-{{ postgrest_x86_release }}.tar.xz" + dest: /tmp/postgrest.tar.xz + checksum: "{{ postgrest_x86_release_checksum }}" + when: platform == "amd64" + +- name: PostgREST - unpack archive in /opt + unarchive: + remote_src: yes + src: /tmp/postgrest.tar.xz + dest: /opt + owner: postgrest + +- name: PostgREST - create service file + template: + src: files/postgrest.service.j2 + dest: /etc/systemd/system/postgrest.service + +- name: PostgREST - reload systemd + systemd: + daemon_reload: yes \ No newline at end of file diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml new file mode 100644 index 000000000..52b69584f --- /dev/null +++ b/ansible/tasks/setup-supabase-internal.yml @@ -0,0 +1,40 @@ +- name: AWS CLI dep + apt: + pkg: + - unzip + - jq + install_recommends: no + +- name: AWS CLI (arm) + get_url: + url: "https://awscli.amazonaws.com/awscli-exe-linux-aarch64-{{ aws_cli_release }}.zip" + dest: "/tmp/awscliv2.zip" + when: platform == "arm64" + +- name: AWS CLI (x86) + get_url: + url: "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-{{ aws_cli_release }}.zip" + dest: "/tmp/awscliv2.zip" + when: platform == "amd64" + +- name: AWS CLI - expand + unarchive: + remote_src: yes + src: "/tmp/awscliv2.zip" + dest: "/tmp" + +- name: AWS CLI - install + shell: "/tmp/aws/install" + become: true + +- name: Install Postgres exporter + import_tasks: internal/postgres-exporter.yml + +- name: Install node exporter + import_tasks: internal/node-exporter.yml + +- name: Install supautils + import_tasks: internal/supautils.yml + +- name: Boot time optimizations + import_tasks: internal/optimizations.yml diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index 05ff49861..3a3db1ad2 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -1,41 +1,34 @@ # DigitalOcean's ubuntu droplet isn't up to date with installed packages, and on # a fresh install I see 71 security upgrades available. +- name: Terminate any ongoing updates + become: yes + shell: killall apt apt-get + ignore_errors: yes + tags: + - update + - update-only + - name: System - apt update and apt upgrade apt: update_cache=yes upgrade=yes # SEE http://archive.vn/DKJjs#parameter-upgrade -- name: Wait for /var/lib/apt/lists/lock +- name: Install required security updates become: yes - shell: while sudo fuser /var/lib/apt/lists/lock; do sleep 10; done; - tags: - - update - - update-only + apt: + pkg: + - tzdata + - linux-libc-dev -- name: Wait for /var/lib/dpkg/lock-frontend +# SEE https://github.com/georchestra/ansible/issues/55#issuecomment-588313638 +# Without this, a similar error is faced +- name: Install Ansible dependencies become: yes - shell: while sudo fuser /var/lib/dpkg/lock-frontend; do sleep 10; done; - tags: - - update - - update-only - -- name: add universe repository for bionic - apt_repository: - repo: deb http://archive.ubuntu.com/ubuntu bionic universe - state: present - -- name: Install python - become: yes apt: pkg: - - python - - python-pip - - python3 - - python3-pip - update_cache: yes - cache_valid_time: 3600 + - acl - name: Install security tools - become: yes + become: yes apt: pkg: - ufw @@ -43,15 +36,24 @@ - unattended-upgrades update_cache: yes cache_valid_time: 3600 - + - name: Adjust APT update intervals - copy: + copy: src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic -- name: Install psycopg2 to enable ansible postgreSQL features - pip: - name: psycopg2-binary +# Find platform architecture and set as a variable +- name: finding platform architecture + shell: if [ $(uname -m) = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi + register: platform_output + tags: + - update + - update-only +- set_fact: + platform: "{{ platform_output.stdout }}" + tags: + - update + - update-only - name: System - Create services.slice template: @@ -59,5 +61,4 @@ dest: /etc/systemd/system/services.slice - name: System - systemd reload - systemd: daemon_reload=yes - + systemd: daemon_reload=yes \ No newline at end of file diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml new file mode 100644 index 000000000..3f2bc5adc --- /dev/null +++ b/ansible/tasks/setup-wal-g.yml @@ -0,0 +1,45 @@ +# Downloading dependencies +- name: Postgres dependencies + become: yes + apt: + pkg: + - liblzo2-dev + - cmake + +# install go dependency for WAL-G +- name: wal-g go dependency + get_url: + url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" + dest: /tmp + +- name: unpack go archive + unarchive: + remote_src: yes + src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz" + dest: /usr/local + +# Download WAL-G +- name: download wal-g + shell: + cmd: go get github.com/wal-g/wal-g; + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + ignore_errors: yes + # ignore error https://github.com/wal-g/wal-g/issues/343#issuecomment-514544288 + +# Install WAL-G +- name: install wal-g + become: yes + shell: + cmd: make install && make deps && make pg_install + chdir: "{{ ansible_env.HOME }}/go/src/github.com/wal-g/wal-g" + environment: + GOBIN: "/usr/local/bin" + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + +# Clean up Go +- name: Uninstall Go + become: yes + file: + path: /usr/local/go + state: absent \ No newline at end of file diff --git a/ansible/vars.yml b/ansible/vars.yml index a80455788..614f595c6 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -1,42 +1,71 @@ -postgresql_version: 12 -postgresql_wal_level: "logical" -postgresql_max_wal_senders: 10 -postgresql_max_replication_slots: 5 -postgresql_row_security: on -postgresql_listen_addresses: - - "*" +supabase_internal: true -postgresql_ext_install_contrib: yes -postgresql_ext_install_dev_headers: yes +postgresql_major: "13" +postgresql_release: "13.3" +postgresql_release_checksum: sha1:aeb645988b1ec9ffbb2fc0a49d9055d3ab17affa -# Warning: Make sure the postgresql & postgis versions are compatible with one another -postgresql_ext_postgis_version: 3 +# Non Postgres Extensions +pgbouncer_release: "1.15.0" +pgbouncer_release_checksum: sha1:ea7e9dbcab178f439a0fa402a78a7f1e4f43e6d4 -postgresql_shared_preload_libraries: [pg_stat_statements, pgaudit, plpgsql, plpgsql_check ] +postgrest_arm_release: 2021-03-05-19-03-d3a8b5f-ubuntu-aarch64 +postgrest_arm_release_checksum: sha1:b9e9b06ead7230b75033e8ae17912714bf463a33 -postgresql_pg_hba_custom: - - {type: "host", database: "all", user: "all", address: "0.0.0.0/0", method: "md5" } +postgrest_x86_release: 2021-03-05-19-03-d3a8b5f-linux-x64-static +postgrest_x86_release_checksum: sha1:4b4adde15f0d41d65a9136d1f8c0d9cd6fe79326 + +aws_cli_release: "2.2.7" + +golang_version: "1.15.4" +wal_g_release: "0.2.15" +wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 + +sfcgal_release: "1.3.10" +sfcgal_release_checksum: sha1:f4add34a00afb0b5f594685fc646565a2bda259b + +postgres_exporter_release: "0.9.0" +postgres_exporter_release_checksum: + arm64: sha256:d869c16791481dc8475487ad84ae4371a63f9b399898ca1c666eead5cccf7182 + amd64: sha256:ff541bd3ee19c0ae003d71424a75edfcc8695e828dd20d5b4555ce433c89d60b + +node_exporter_release: 1.1.2 +node_exporter_release_checksum: + arm64: sha256:eb5e7d16f18bb3272d0d832986fc8ac6cb0b6c42d487c94e15dabb10feae8e04 + amd64: sha256:8c1f6a317457a658e0ae68ad710f6b4098db2cad10204649b51e3c043aa3e70d + +# Postgres Extensions +postgis_release: "3.1.2" +postgis_release_checksum: sha1:622f52f3bf338c8e51ea6d73d30d6a5d3140c517 + +pgrouting_release: "3.2.0" +pgrouting_release_checksum: sha1:d902d449ebc96b6cdcb2fac09434d0098467cda5 pgtap_release: "1.1.0" pgtap_release_checksum: sha1:cca57708e723de18735a723b774577dc52f6f31e -pgaudit_release: "1.4.0" -pgaudit_release_checksum: sha1:ea085fbf227b5c461331ab33b99579f37db299a6 +pg_cron_release: "1.3.1" +pg_cron_release_checksum: sha1:679b6ff54e0b1070a5fd713c5d25c3378f371fac + +pgaudit_release: "1.5.0" +pgaudit_release_checksum: sha1:8429125e8f70fcaa2c2f5a0e22b910a4afb821a4 pgsql_http_release: "1.3.1" pgsql_http_release_checksum: sha1:816a3fff53e05301b176cf0696799fc5a00f54e8 -plpgsql_check_release: "1.11.0" -plpgsql_check_release_checksum: sha1:395313b6ef9c10c4fc182817d6f0040b171147b8 +plpgsql_check_release: "1.16.0" +plpgsql_check_release_checksum: sha1:626553fc2746fe10aa5a776a1229baf2af3365fc -pljava_release: "1_5_5" -pljava_release_checksum: sha1:5277433030fdeed8528c7c0154163b54aedbd842 +pg_safeupdate_release: "1.3" +pg_safeupdate_release_checksum: sha1:34a0353611bfd63f7ea760aac2afcb518bf3ba7c -postgresql_log_destination: "csvlog" -postgresql_logging_collector: on -postgresql_log_filename: "postgresql.log" -postgresql_log_rotation_age: 0 -postgresql_log_rotation_size: 0 +timescaledb_release: "2.3.0" + +wal2json_release: "2_3" +wal2json_release_checksum: sha1:923f9bbcd0505a1f0b6eac1d371e4ff2d266a958 + +supautils_release: "1.1.0" +supautils_release_checksum: sha1:326ac5c1933bd30d4a50da7568b27629a9ec544b + +pljava_release: "1_6_2" +pljava_release_checksum: sha1:9610b80cbd13d4d43bcdaa2928365dbfd1bf6e94 -wal_g_release: "0.2.15" -wal_g_release_checksum: sha1:e82d405121e0ccc322a323b9824e60c102b14004 \ No newline at end of file diff --git a/digitalOcean.json b/digitalOcean.json index 5c3d64b4a..36396fc29 100644 --- a/digitalOcean.json +++ b/digitalOcean.json @@ -1,22 +1,31 @@ { "variables": { "do_token": "", - "image_name": "ubuntu-18-04-x64", + "image_name": "ubuntu-20-04-x64", "region": "sgp1", "snapshot_regions": "sgp1", - "ansible_arguments": "--skip-tags,update-only" + "snapshot_name": "supabase-postgres-13.3.0", + "ansible_arguments": "--skip-tags,update-only,--skip-tags,aws-only,-e,supabase_internal='false'" }, - "builders": [{ - "type": "digitalocean", - "api_token": "{{user `do_token`}}", - "image": "{{user `image_name`}}", - "region": "{{user `region`}}", - "snapshot_regions": "{{user `snapshot_regions`}}", - "size": "s-1vcpu-1gb", - "ssh_username": "root", - "snapshot_name": "supabase-postgres-0.13.0" - }], + "builders": [ + { + "type": "digitalocean", + "api_token": "{{user `do_token`}}", + "image": "{{user `image_name`}}", + "region": "{{user `region`}}", + "snapshot_regions": "{{user `snapshot_regions`}}", + "size": "s-1vcpu-1gb", + "ssh_username": "root", + "snapshot_name": "{{user `snapshot_name`}}" + } + ], "provisioners": [ + { + "type": "shell", + "inline": [ + "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done" + ] + }, { "type": "ansible", "user": "root", @@ -26,11 +35,11 @@ { "type": "shell", "scripts": [ - "scripts/01-test", + "scripts/01-postgres_check.sh", "scripts/90-cleanup.sh", "scripts/91-log_cleanup.sh", "scripts/99-img_check.sh" ] } ] -} \ No newline at end of file +} diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index b0b285619..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,180 +0,0 @@ -FROM postgres:12 - -# install postgis -ENV POSTGIS_MAJOR 3 -ENV POSTGIS_VERSION 3.0.0+dfsg-2~exp1.pgdg100+1 -RUN apt-get update \ - && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ - && apt-get install -y --no-install-recommends \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ - && rm -rf /var/lib/apt/lists/* /var/tmp/* - -# install pgtap -ENV PGTAP_VERSION=v1.1.0 - -RUN pgtapDependencies="git \ - ca-certificates \ - build-essential" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${pgtapDependencies} \ - && cd /tmp \ - && git clone git://github.com/theory/pgtap.git \ - && cd pgtap \ - && git checkout tags/$PGTAP_VERSION \ - && make install \ - && apt-get clean \ - && apt-get remove -y ${pgtapDependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/pgtap /var/lib/apt/lists/* /var/tmp/* - -# install plpython3 -RUN apt-get update \ - && apt-get install postgresql-plpython3-12 -y - -# install pgAudit -ENV PGAUDIT_VERSION=1.4.0 - -RUN pgAuditDependencies="git \ - ca-certificates \ - build-essential \ - postgresql-server-dev-$PG_MAJOR \ - libssl-dev \ - libkrb5-dev" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${pgAuditDependencies} \ - && cd /tmp \ - && git clone https://github.com/pgaudit/pgaudit.git \ - && cd pgaudit \ - && git checkout ${PGAUDIT_VERSION} \ - && make check USE_PGXS=1 \ - && make install USE_PGXS=1 \ - && apt-get clean \ - && apt-get remove -y ${pgAuditDependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/pgaudit /var/lib/apt/lists/* /var/tmp/* - -# install pgjwt -RUN pgjwtDependencies="git \ - ca-certificates \ - build-essential" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${pgjwtDependencies} \ - && cd /tmp \ - && git clone https://github.com/michelp/pgjwt.git \ - && cd pgjwt \ - && git checkout master \ - && make install \ - && apt-get clean \ - && apt-get remove -y ${pgtapDependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/pgjwt /var/lib/apt/lists/* /var/tmp/* - -# install pgsql-http -ENV PGSQL_HTTP_VERSION=v1.3.1 - -RUN pgsqlHttpDependencies="git \ - ca-certificates \ - build-essential \ - postgresql-server-dev-$PG_MAJOR" \ - && pgsqlHttpRuntimeDependencies="libcurl4-gnutls-dev" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${pgsqlHttpDependencies} ${pgsqlHttpRuntimeDependencies} \ - && cd /tmp \ - && git clone https://github.com/pramsey/pgsql-http.git \ - && cd pgsql-http \ - && git checkout ${PGSQL_HTTP_VERSION} \ - && make \ - && make install \ - && apt-get clean \ - && apt-get remove -y ${pgsqlHttpDependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/pgsql-http /var/lib/apt/lists/* /var/tmp/* - -# install plpgsql_check -ENV PLPGSQL_CHECK_VERSION=v1.11.3 - -RUN plpgsqlCheckDependencies="git \ - ca-certificates \ - build-essential \ - postgresql-server-dev-$PG_MAJOR" \ - && plpgsqlCheckRuntimeDependencies="libicu-dev" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${plpgsqlCheckDependencies} ${plpgsqlCheckRuntimeDependencies} \ - && cd /tmp \ - && git clone https://github.com/okbob/plpgsql_check.git \ - && cd plpgsql_check \ - && git checkout ${PLPGSQL_CHECK_VERSION} \ - && make clean \ - && make install \ - && apt-get clean \ - && apt-get remove -y ${pgsqlHttpDependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/plpgsql_check /var/lib/apt/lists/* /var/tmp/* - -# install plv8 -ENV PLV8_VERSION=r3.0alpha - -RUN plv8Dependencies="build-essential \ - ca-certificates \ - curl \ - git-core \ - python \ - gpp \ - cpp \ - pkg-config \ - apt-transport-https \ - cmake \ - libc++-dev \ - libc++abi-dev \ - postgresql-server-dev-$PG_MAJOR" \ - && plv8RuntimeDependencies="libc++1 \ - libtinfo5 \ - libc++abi1" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${plv8Dependencies} ${plv8RuntimeDependencies} \ - && mkdir -p /tmp/build \ - && cd /tmp/build \ - && git clone https://github.com/plv8/plv8.git \ - && cd plv8 \ - && git checkout ${PLV8_VERSION} \ - && make static \ - && make install \ - && rm -rf /root/.vpython_cipd_cache /root/.vpython-root \ - && apt-get clean \ - && apt-get remove -y ${plv8Dependencies} \ - && apt-get autoremove -y \ - && rm -rf /tmp/build /var/lib/apt/lists/* /var/tmp/* - -# install pljava -ENV PLJAVA_VERSION=V1_5_5 - -RUN pljavaDependencies="git \ - ca-certificates \ - g++ \ - maven \ - postgresql-server-dev-$PG_MAJOR \ - libpq-dev \ - libecpg-dev \ - libkrb5-dev \ - default-jdk \ - libssl-dev" \ - && apt-get update \ - && apt-get install -y --no-install-recommends ${pljavaDependencies} \ - && cd /tmp \ - && git clone https://github.com/tada/pljava.git \ - && cd pljava \ - && git checkout ${PLJAVA_VERSION} \ - && mvn clean install \ - && java -jar pljava-packaging/target/pljava-pg12.3-amd64-Linux-gpp.jar \ - && apt-get clean \ - && apt-get remove -y ${pljavaDependencies} \ - && apt-get autoremove -y \ - && rm -rf ~/.m2 /tmp/pljava /var/lib/apt/lists/* /var/tmp/* - -RUN apt-get update \ - && apt-get install -y --no-install-recommends default-jdk-headless \ - && rm -rf /var/lib/apt/lists/* /var/tmp/* - -RUN mkdir -p /docker-entrypoint-initdb.d -ADD ./mnt /docker-entrypoint-initdb.d/ \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 09df67353..8775d6b8a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: db: @@ -6,4 +6,4 @@ services: ports: - "5432:5432" environment: - POSTGRES_PASSWORD: postgres \ No newline at end of file + POSTGRES_PASSWORD: postgres diff --git a/docker/mnt/init-permissions.sh b/docker/mnt/init-permissions.sh deleted file mode 100644 index 314d387d0..000000000 --- a/docker/mnt/init-permissions.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -echo "host replication $POSTGRES_USER 0.0.0.0/0 trust" >> $PGDATA/pg_hba.conf -echo "shared_preload_libraries = 'pg_stat_statements, pgaudit'" >> $PGDATA/postgresql.conf -echo "pg_stat_statements.max = 10000" >> $PGDATA/postgresql.conf -echo "pg_stat_statements.track = all" >> $PGDATA/postgresql.conf -echo "wal_level=logical" >> $PGDATA/postgresql.conf -echo "max_replication_slots=5" >> $PGDATA/postgresql.conf -echo "max_wal_senders=10" >> $PGDATA/postgresql.conf -echo "log_destination='csvlog'" >> $PGDATA/postgresql.conf -echo "logging_collector=on" >> $PGDATA/postgresql.conf -echo "log_filename='postgresql.log'" >> $PGDATA/postgresql.conf -echo "log_rotation_age=0" >> $PGDATA/postgresql.conf -echo "log_rotation_size=0" >> $PGDATA/postgresql.conf -echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-amd64/lib/server/libjvm.so'" >> $PGDATA/postgresql.conf diff --git a/rfcs/0001-connection-pooling.md b/rfcs/0001-connection-pooling.md new file mode 100644 index 000000000..e66a19c5d --- /dev/null +++ b/rfcs/0001-connection-pooling.md @@ -0,0 +1,71 @@ +--- +feature: Connection Pooling +start-date: 2021-02-04 +author: kiwicopple +co-authors: steve-chavez, dragarcia +related-issues: (will contain links to implementation PRs) +--- + +# Summary +[summary]: #summary + +We would like to explore connection pooling on Supabase. This RFC is intended to decide: + +- Whether we should provide a pooler +- Which connection pooler we should use +- Where in the stack it would be installed - i.e. if should bundle it with the Postgres build + + +# Motivation +[motivation]: #motivation + +In Postgres, every connection is a process. Because of this, a lot of connections to the database can be very expensive on memory. + +When connecting to Postgres database from serverless functions, there is no connection pooling, and so the server needs to maintain hundreds/thousands of connections. + + +# Detailed design +[design]: #detailed-design + +This is still in the "Gather Feedback" stage. To start the discussion: + + +### 1. Decide on a PG Pooler + +- `pg_bouncer` - https://www.pgbouncer.org/ +- `PG Pool II` - https://www.pgpool.net/mediawiki/index.php/Main_Page +- `odyssey` - https://github.com/yandex/odyssey +- others? + +### 2. Decide on configuration + +Most poolers allow different configurations. We would need to decide on how we would configure the pooler by default + +### 3. Decide if the user should be able re-configure the pooler + +Should a user be able to change the configuration? If so, how would they do it? + + +# Drawbacks +[drawbacks]: #drawbacks + +- Security +- Not directly relevant to the "supabase" stack, so it's additional non-core support + +# Alternatives +[alternatives]: #alternatives + +1. Since we already offer [PostgREST](https://github.com/postgrest/postgrest) and [postgres-meta](https://github.com/supabase/pg-api), this isn't entirely necessary for the Supabase stack. Bundling this is only beneficial for connecting external tools. +2. We could hold back on this implementation until we move to a full Postgres Operator, which would include a pooler. It would be nice to have something for local development though. + + +# Unresolved questions +[unresolved]: #unresolved-questions + +- Add any unresolved questions here + + +# Future work +[future]: #future-work + +- Add any future work here \ No newline at end of file diff --git a/scripts/01-postgres_check.sh b/scripts/01-postgres_check.sh new file mode 100644 index 000000000..d131528ee --- /dev/null +++ b/scripts/01-postgres_check.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Scripts in this directory are run during the build process. +# each script will be uploaded to /tmp on your build droplet, +# given execute permissions and run. The cleanup process will +# remove the scripts from your build system after they have run +# if you use the build_image task. +# +echo "Commencing Checks" + +function check_database_is_ready { + echo -e "\nChecking if database is ready and accepting connections:" + if [ "$(pg_isready)" = "/tmp:5432 - accepting connections" ]; then + echo "Database is ready" + else + echo "Error: Database is not ready. Exiting" + exit 1 + fi +} + +function check_postgres_owned_dir_exists { + DIR=$1 + USER="postgres" + + echo -e "\nChecking if $DIR exists and owned by postgres user:" + + if [ -d "$DIR" ]; then + echo "$DIR exists" + if [ $(stat -c '%U' $DIR) = "$USER" ]; then + echo "$DIR is owned by $USER" + else + echo "Error: $DIR is not owned by $USER" + exit 1 + fi + else + echo "Error: ${DIR} not found. Exiting." + exit 1 + fi +} + +function check_lse_enabled { + ARCH=$(uname -m) + if [ $ARCH = "aarch64" ]; then + echo -e "\nArchitecture is $ARCH. Checking for LSE:" + + LSE_COUNT=$(objdump -d /usr/lib/postgresql/bin/postgres | grep -i 'ldxr\|ldaxr\|stxr\|stlxr' | wc -l) + MOUTLINE_ATOMICS_COUNT=$(nm /usr/lib/postgresql/bin/postgres | grep __aarch64_have_lse_atomics | wc -l) + + # Checking for load and store exclusives + if [ $LSE_COUNT -gt 0 ]; then + echo "Postgres has LSE enabled" + else + echo "Error: Postgres failed to be compiled with LSE. Exiting" + exit 1 + fi + + # Checking if successfully compiled with -moutline-atomics + if [ $MOUTLINE_ATOMICS_COUNT -gt 0 ]; then + echo "Postgres has been compiled with -moutline-atomics" + else + echo "Error: Postgres failed to be compiled with -moutline-atomics. Exiting" + exit 1 + fi + else + echo "Architecture is $ARCH. Not checking for LSE." + fi +} + +check_database_is_ready +check_postgres_owned_dir_exists "/var/lib/postgresql" +check_postgres_owned_dir_exists "/etc/postgresql" +check_lse_enabled \ No newline at end of file diff --git a/scripts/01-test b/scripts/01-test deleted file mode 100644 index e5b3e0523..000000000 --- a/scripts/01-test +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# -# Scripts in this directory are run during the build process. -# each script will be uploaded to /tmp on your build droplet, -# given execute permissions and run. The cleanup process will -# remove the scripts from your build system after they have run -# if you use the build_image task. -# -echo "Commencing Digital Ocean Checks" diff --git a/scripts/11-lemp.sh b/scripts/11-lemp.sh new file mode 100644 index 000000000..c340f5e9f --- /dev/null +++ b/scripts/11-lemp.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# DigitalOcean Marketplace Image Validation Tool +# © 2021 DigitalOcean LLC. +# This code is licensed under Apache 2.0 license (see LICENSE.md for details) + +rm -rvf /etc/nginx/sites-enabled/default + +ln -s /etc/nginx/sites-available/digitalocean \ + /etc/nginx/sites-enabled/digitalocean + +rm -rf /var/www/html/index*debian.html + +chown -R www-data: /var/www \ No newline at end of file diff --git a/scripts/12-ufw-nginx.sh b/scripts/12-ufw-nginx.sh new file mode 100644 index 000000000..7c47366cd --- /dev/null +++ b/scripts/12-ufw-nginx.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# DigitalOcean Marketplace Image Validation Tool +# © 2021 DigitalOcean LLC. +# This code is licensed under Apache 2.0 license (see LICENSE.md for details) + +ufw limit ssh +ufw allow 'Nginx Full' + +ufw --force enable \ No newline at end of file diff --git a/scripts/13-force-ssh-logout.sh b/scripts/13-force-ssh-logout.sh new file mode 100644 index 000000000..99e28c180 --- /dev/null +++ b/scripts/13-force-ssh-logout.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# DigitalOcean Marketplace Image Validation Tool +# © 2021 DigitalOcean LLC. +# This code is licensed under Apache 2.0 license (see LICENSE.md for details) + +cat >> /etc/ssh/sshd_config < /root/.bash_history unset HISTFILE -apt-get -y autoremove -apt-get -y autoclean find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? rm -rf /var/lib/cloud/instances/* @@ -33,4 +53,4 @@ dd if=/dev/zero of=/zerofile & sleep 5 done sync; rm /zerofile; sync -cat /dev/null > /var/log/lastlog; cat /dev/null > /var/log/wtmp +cat /dev/null > /var/log/lastlog; cat /dev/null > /var/log/wtmp \ No newline at end of file diff --git a/scripts/91-log_cleanup.sh b/scripts/91-log_cleanup.sh index 26f5fbcc9..85211640a 100644 --- a/scripts/91-log_cleanup.sh +++ b/scripts/91-log_cleanup.sh @@ -3,3 +3,9 @@ # echo "Clearing all log files" rm -rf /var/log/* + +# https://github.com/fail2ban/fail2ban/issues/1593 +touch /var/log/auth.log + +touch /var/log/pgbouncer.log +chown postgres:postgres /var/log/pgbouncer.log \ No newline at end of file diff --git a/scripts/99-img_check.sh b/scripts/99-img_check.sh index 6daee6871..00b547641 100755 --- a/scripts/99-img_check.sh +++ b/scripts/99-img_check.sh @@ -1,10 +1,10 @@ #!/bin/bash -# + # DigitalOcean Marketplace Image Validation Tool -# © 2018 DigitalOcean LLC. -# This code is licensed under MIT license (see LICENSE.txt for details) -# -VERSION="v. 1.2" +# © 2021 DigitalOcean LLC. +# This code is licensed under Apache 2.0 license (see LICENSE.md for details) + +VERSION="v. 1.6" RUNDATE=$( date ) # Script should be run with SUDO @@ -110,18 +110,12 @@ function checkLogs { [[ -e $f ]] || break if [[ "${f}" = '/var/log/lfd.log' && "$( cat "${f}" | egrep -v '/var/log/messages has been reset| Watching /var/log/messages' | wc -c)" -gt 50 ]]; then if [ $f != $cp_ignore ]; then - echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found\n" - ((WARN++)) - if [[ $STATUS != 2 ]]; then - STATUS=1 - fi - fi - elif [[ "${f}" == '/var/log/cloud-init-output.log' ]]; then - if cat '/var/log/cloud-init-output.log' | grep -q SHA256; then - echo -en "\e[41m[FAIL]\e[0m log containing SHA256 value found in log file ${f}\n" - ((FAIL++)) - STATUS=1 + echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found\n" + ((WARN++)) + if [[ $STATUS != 2 ]]; then + STATUS=1 fi + fi elif [[ "${f}" != '/var/log/lfd.log' && "$( cat "${f}" | wc -c)" -gt 50 ]]; then if [ $f != $cp_ignore ]; then echo -en "\e[93m[WARN]\e[0m un-cleared log file, ${f} found\n" @@ -252,7 +246,7 @@ function checkUsers { echo -en "\e[32m[PASS]\e[0m User ${user} has no password set.\n" ((PASS++)) else - echo -en "\e[41m[FAIL]\e[0m User ${user} has a password set on their account.\n" + echo -en "\e[41m[FAIL]\e[0m User ${user} has a password set on their account. Only system users are allowed on the image.\n" ((FAIL++)) STATUS=2 fi @@ -385,7 +379,7 @@ function checkFirewall { # we will check some of the most common if cmdExists 'ufw'; then fw="ufw" - ufwa=$(ufw status | sed -e "s/^Status:\ //") + ufwa=$(ufw status |head -1| sed -e "s/^Status:\ //") if [[ $ufwa == "active" ]]; then FW_VER="\e[32m[PASS]\e[0m Firewall service (${fw}) is active\n" ((PASS++)) @@ -418,6 +412,14 @@ function checkFirewall { } function checkUpdates { if [[ $OS == "Ubuntu" ]] || [[ "$OS" =~ Debian.* ]]; then + # Ensure /tmp exists and has the proper permissions before + # checking for security updates + # https://github.com/digitalocean/marketplace-partners/issues/94 + if [[ ! -d /tmp ]]; then + mkdir /tmp + fi + chmod 1777 /tmp + echo -en "\nUpdating apt package database to check for security updates, this may take a minute...\n\n" apt-get -y update > /dev/null @@ -441,11 +443,11 @@ function checkUpdates { echo -en "\e[32m[PASS]\e[0m There are no pending security updates for this image.\n\n" fi elif [[ $OS == "CentOS Linux" ]]; then - echo -en "\nChecking for available updates with yum, this may take a minute...\n\n" + echo -en "\nChecking for available security updates, this may take a minute...\n\n" - update_count=$(yum list updates -q | grep -vc "Updated Packages") + update_count=$(yum check-update --security --quiet | wc -l) if [[ $update_count -gt 0 ]]; then - echo -en "\e[41m[FAIL]\e[0m There are ${update_count} updates available for this image that have not been installed.\n" + echo -en "\e[41m[FAIL]\e[0m There are ${update_count} security updates available for this image that have not been installed.\n" ((FAIL++)) STATUS=2 else @@ -567,7 +569,9 @@ osv=0 if [[ $OS == "Ubuntu" ]]; then ost=1 - if [[ $VER == "18.04" ]]; then + if [[ $VER == "20.04" ]]; then + osv=1 + elif [[ $VER == "18.04" ]]; then osv=1 elif [[ $VER == "16.04" ]]; then osv=1 @@ -591,7 +595,9 @@ elif [[ "$OS" =~ Debian.* ]]; then elif [[ $OS == "CentOS Linux" ]]; then ost=1 - if [[ $VER == "7" ]]; then + if [[ $VER == "8" ]]; then + osv=1 + elif [[ $VER == "7" ]]; then osv=1 elif [[ $VER == "6" ]]; then osv=1 @@ -669,8 +675,8 @@ if [[ $STATUS == 0 ]]; then exit 0 elif [[ $STATUS == 1 ]]; then echo -en "Please review all [WARN] items above and ensure they are intended or resolved. If you do not have a specific requirement, we recommend resolving these items before image submission\n\n" - exit 1 + exit 0 else - echo -en "Some critical tests failed. These items must be resolved and this scan re-run before you submit your image to the marketplace.\n\n" + echo -en "Some critical tests failed. These items must be resolved and this scan re-run before you submit your image to the DigitalOcean Marketplace.\n\n" exit 1 -fi +fi \ No newline at end of file