Merge pull request #75 from ZPascal/issue-74

feat: Add the possibility to specify the used PostgreSQL version

README.md

@@ -76,6 +76,7 @@ The complete list of available properties can be found in the [spec](jobs/postgr
 
 Property | Description
 -------- | -------------
+databases.version | Define the used PostgreSQL major version. Default: 16
 databases.port | The database port. Default: 5432
 databases.databases | A list of databases and associated properties to create when Postgres starts
 databases.databases[n].name | Database name

ci/pipeline.yml

@@ -86,6 +86,18 @@ jobs:
             options:
               access_key_id: ((postgres-release-blobstore-user.username))
               secret_access_key: ((postgres-release-blobstore-user.password))
+    - task: bump-yq-package
+      file: postgres-release/ci/tasks/bump-yq-packages/task.yml
+      image: bosh-cli-registry-image
+      input_mapping:
+        yq-src: postgres-yq-4-src
+      params:
+        MAJOR_VERSION: 4
+        PRIVATE_YML: |
+          blobstore:
+            options:
+              access_key_id: ((postgres-release-blobstore-user.username))
+              secret_access_key: ((postgres-release-blobstore-user.password))
     - put: postgres-release
       params:
         repository: postgres-release
@@ -232,6 +244,13 @@ resources:
       regex: 'href="v(?P<version>16\.[0-9.]+)/"'
       uri: "https://ftp.postgresql.org/pub/source/v{version}/postgresql-{version}.tar.gz"
 
+  - name: postgres-yq-4-src
+    type: http-resource
+    source:
+      index: "https://api.github.com/repos/mikefarah/yq/releases"
+      regex: '"tag_name":"v(?P<version>4\.[0-9.]+)/"'
+      uri: "https://github.com/mikefarah/yq/releases/download/v{version}/yq_linux_amd64"
+
   - name: postgres-release
     type: git
     source:
@@ -263,4 +282,3 @@ resource_types:
     type: docker-image
     source:
       repository: aequitas/http-resource
-

ci/tasks/bump-postgres-packages/task.sh

@@ -19,14 +19,12 @@ pushd postgres-release
   fi
 
   if ${NEED_COMMIT}; then
-    eval $(grep "^current_version=" jobs/postgres/templates/pgconfig.sh.erb)
-    new_version=$(echo "${FILENAME}" | egrep -o "[0-9.]+[0-9]")
+    latest_yq_version=$(curl -s -L https://api.github.com/repos/mikefarah/yq/releases/latest | grep "tag_name" | sed s/\"tag_name\":\//g | sed s/\"//g | sed s/\,//g | sed s/v//g | xargs)
+    curl -s -L https://github.com/mikefarah/yq/releases/download/v${latest_yq_version}/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq
 
-    # If new version is greater than current version, update the current_version variables
-    if printf '%s\n' "$current_version" "$new_version" | sort -V -C; then
-      sed -i "s/^current_version=.*/current_version=\"${new_version}\"/" jobs/postgres/templates/pgconfig.sh.erb
-      sed -i "s/^current_version=.*/current_version=\"${new_version}\"/" jobs/bbr-postgres-db/templates/config.sh.erb
-    fi
+    echo "-----> $(date): Update the PostgreSQL version inside the used_postgresql_versions.yml file"
+    current_minor_version=$(cat config/blobs.yml  | grep "postgresql-${MAJOR_VERSION}" | cut -f1 -d: | sed "s/postgres\/postgresql-//g" | sed "s/.tar.gz//g")
+    CURRENT_MINOR_VERSION=$current_minor_version /tmp/yq -i '.postgresql.major_version[env(MAJOR_VERSION)].minor_version = strenv(CURRENT_MINOR_VERSION)' jobs/postgres/config/used_postgresql_versions.yml
 
     echo "-----> $(date): Creating git commit"
     git config user.name "$GIT_USER_NAME"

ci/tasks/bump-yq-packages/task.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+NEED_COMMIT=false
+
+set -euo pipefail
+
+echo "${PRIVATE_YML}" > postgres-release/config/private.yml
+
+pushd postgres-release
+  CURRENT_BLOBS=$(bosh blobs)
+  CURRENT_VERSION=$(cat ../yq-src/version)
+  mv ../yq-src/yq_linux_amd64 ../yq-src/postgres-yq-${CURRENT_VERSION}
+  BLOB_PATH=$(ls ../yq-src/postgres-yq-${CURRENT_VERSION})
+  FILENAME=$( basename ${BLOB_PATH} )
+  OLD_BLOB_PATH=$(cat config/blobs.yml  | grep "postgres-yq-${MAJOR_VERSION}" | cut -f1 -d:)
+  if ! echo "${CURRENT_BLOBS}" | grep "${FILENAME}" ; then
+    NEED_COMMIT=true
+    echo "adding ${FILENAME}"
+    bosh add-blob --sha2 "${BLOB_PATH}" "yq/postgres-${FILENAME}"
+    bosh remove-blob ${OLD_BLOB_PATH}
+    bosh upload-blobs
+  fi
+
+  if ${NEED_COMMIT}; then
+    echo "-----> $(date): Creating git commit"
+    git config user.name "$GIT_USER_NAME"
+    git config user.email "$GIT_USER_EMAIL"
+    git add .
+
+    git --no-pager diff --cached
+    if [[ "$( git status --porcelain )" != "" ]]; then
+      git commit -am "Bump packages"
+    fi
+  fi
+popd

ci/tasks/bump-yq-packages/task.yml

@@ -0,0 +1,17 @@
+---
+platform: linux
+
+inputs:
+- name: yq-src
+
+outputs:
+- name: postgres-release
+
+run:
+  path: postgres-release/ci/tasks/bump-yq-packages/task.sh
+
+params:
+  PRIVATE_YML:
+  MAJOR_VERSION:
+  GIT_USER_NAME: CI Bot
+  GIT_USER_EMAIL: bots@cloudfoundry.org

ci/tasks/create-final-release/task.sh

@@ -8,11 +8,15 @@ set +x
 echo "$RELEASE_PRIVATE_YML" >> "config/private.yml"
 set -x
 
+latest_yq_version=$(curl -s -L https://api.github.com/repos/mikefarah/yq/releases/latest | grep "tag_name" | sed s/\"tag_name\":\//g | sed s/\"//g | sed s/\,//g | sed s/v//g | xargs)
+curl -s -L https://github.com/mikefarah/yq/releases/download/v${latest_yq_version}/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq
+
 bosh -n create-release --final
 new_release_version="$(find releases -regex ".*postgres-[0-9]*.yml" | egrep -o "[0-9]+" | sort -n | tail -n 1)"
 
-eval $(grep "^current_version=" jobs/postgres/templates/pgconfig.sh.erb)
-sed -i "/^versions:/a\ \ ${new_release_version}: \"PostgreSQL ${current_version}\"" versions.yml
+current_major_version=$(/tmp/yq '.postgresql.default' jobs/postgres/config/used_postgresql_versions.yml)
+current_minor_version=$(CURRENT_MAJOR_VERSION=$current_major_version /tmp/yq '.postgresql.major_version[env(CURRENT_MAJOR_VERSION)].minor_version' jobs/postgres/config/used_postgresql_versions.yml)
+sed -i "/^versions:/a\ \ ${new_release_version}: \"PostgreSQL ${current_minor_version}\"" versions.yml
 git add versions.yml
 
 git add .final_builds releases

ci/tasks/run-acceptance-tests/task.sh

@@ -2,6 +2,11 @@
 
 root="${PWD}"
 
+function setup_yq() {
+    latest_yq_version=$(curl -s -L https://api.github.com/repos/mikefarah/yq/releases/latest | grep "tag_name" | sed s/\"tag_name\":\//g | sed s/\"//g | sed s/\,//g | sed s/v//g | xargs)
+    curl -s -L https://github.com/mikefarah/yq/releases/download/v${latest_yq_version}/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq
+}
+
 function setup_bosh() {
   source start-bosh
   source /tmp/local-bosh/director/env
@@ -43,10 +48,13 @@ function upload_release() {
 
 function main() {
   setup_bosh
+  setup_yq
   install_bbr
   upload_release
   bosh upload-stemcell stemcell/stemcell.tgz
-  cat ${root}/postgres-release/jobs/postgres/templates/pgconfig.sh.erb | grep current_version > ${root}/pgconfig.sh
+  current_major_version=$(/tmp/yq '.postgresql.default' jobs/postgres/config/used_postgresql_versions.yml)
+  current_minor_version=$(CURRENT_MAJOR_VERSION=$current_major_version /tmp/yq '.postgresql.major_version[env(CURRENT_MAJOR_VERSION)].minor_version' jobs/postgres/config/used_postgresql_versions.yml)
+  echo "current_version=${current_minor_version}" > ${root}/pgconfig.sh
   source ${root}/pgconfig.sh
   config_file="${root}/pgats_config.yml"
   create_config_file > $config_file

jobs/bbr-postgres-db/spec

@@ -18,6 +18,7 @@ packages:
   - postgres-13
   - postgres-15
   - postgres-16
+  - postgres-yq-4
 
 consumes:
 - name: database

jobs/bbr-postgres-db/templates/config.sh.erb

@@ -22,9 +22,11 @@ if_link("database") do |data|
 end
 
  %>
-current_version="16.3"
+
 JOB_DIR="/var/vcap/jobs/bbr-postgres-db"
-PACKAGE_DIR="/var/vcap/packages/postgres-${current_version%.*}"
+current_version=postgres-$(MAJOR_VERSION=<%=p('databases.version')%> ${JOB_DIR}/packages/postgres-yq-4/bin/yq '.postgresql.major_version[env(MAJOR_VERSION)].minor_version' /var/vcap/jobs/postgres/config/used_postgresql_versions.yml)
+
+PACKAGE_DIR="/var/vcap/packages/${current_version%.*}"
 PORT="<%= port %>"
 DATABASES=(<%= databases.map{|d| d["name"]}.join(' ')%>)
 DBHOST=<%= dbhost %>

jobs/postgres/spec

@@ -24,13 +24,15 @@ templates:
   hooks/postgres-pre-stop.sh.erb: bin/hooks/postgres-pre-stop.sh
   hooks/postgres-post-start.sh.erb: bin/hooks/postgres-post-start.sh
   hooks/postgres-post-stop.sh.erb: bin/hooks/postgres-post-stop.sh
+  used_postgresql_versions.yml: config/used_postgresql_versions.yml
 
 packages:
   - postgres-common
   - postgres-11
   - postgres-13
   - postgres-15
   - postgres-16
+  - postgres-yq-4
 
 provides:
 - name: postgres
@@ -42,6 +44,9 @@ provides:
   - databases.tls.ca
 
 properties:
+  databases.version:
+    description: "The database version e.g. 11, 13, 15 or 16"
+    default: 16
   databases.port:
     description: "The database port"
     default: 5432

jobs/postgres/templates/pgconfig.sh.erb

@@ -6,10 +6,10 @@ set -x # if you want tracing disabled, set 'databases.enable_traces: false' in t
 ENABLE_TRACE=0
 # set -x # uncomment it if you want to enable tracing in all control scripts
 <% end %>
-current_version="16.3"
-pgversion_current="postgres-${current_version}"
-
 JOB_DIR=/var/vcap/jobs/postgres
+current_version="<%=p('databases.version')%>"
+pgversion_current=postgres-$(MAJOR_VERSION=<%=p('databases.version')%> ${JOB_DIR}/packages/postgres-yq-4/bin/yq '.postgresql.major_version[env(MAJOR_VERSION)].minor_version' /var/vcap/jobs/postgres/config/used_postgresql_versions.yml)
+
 PACKAGE_DIR="/var/vcap/packages/${pgversion_current%.*}"
 STORE_DIR=/var/vcap/store
 PG_STORE_DIR=${STORE_DIR}/postgres

jobs/postgres/templates/used_postgresql_versions.yml

@@ -0,0 +1,11 @@
+postgresql:
+  default: 16
+  major_version:
+    16:
+      minor_version: "16.3"
+    15:
+      minor_version: "15.6"
+    13:
+      minor_version: "13.14"
+    11:
+      minor_version: "11.22"

jobs/postgres/templates/utils.sh.erb

@@ -68,6 +68,13 @@ function is_major() {
   [ "${pgversion_current%.*}" != "${pgversion_upgrade_from%.*}" ]
 }
 
+function check_postgresql_versions(){
+    if [[ $(echo -e "$pgversion_upgrade_from\n$pgversion_current" | sort --version-sort | head --lines=1) != $pgversion_upgrade_from ]]; then
+        echo "The downgrade of the database instance is not supported."
+        exit 1
+    fi
+}
+
 function init_data_dir(){
   if [ ! -f "${DATA_DIR}/postgresql.conf" ]; then
     # initdb creates data directories
@@ -77,7 +84,7 @@ function init_data_dir(){
 }
 
 function run_major_upgrade(){
-  if is_major; then
+  if is_major && check_postgresql_versions; then
     rm -rf ${DATA_DIR_PREVIOUS}
     echo "Running a PostgreSQL major upgrade from ${pgversion_upgrade_from} to ${pgversion_current}"
     touch "$POSTGRES_UPGRADE_LOCK"
@@ -105,7 +112,7 @@ EOF
   fi
 }
 function run_minor_upgrade(){
-  if ! is_major; then
+  if ! is_major && check_postgresql_versions; then
     rm -rf ${DATA_DIR_PREVIOUS}
     echo "Running a PostgreSQL minor upgrade from ${pgversion_upgrade_from} to ${pgversion_current}"
     <% if p("databases.skip_data_copy_in_minor") %>

packages/postgres-yq-4/packaging

@@ -0,0 +1,17 @@
+#!/bin/bash -exu
+
+function main() {
+  install
+
+}
+
+function install() {
+
+  echo "Install yq..."
+  mkdir -p "${BOSH_INSTALL_TARGET}/bin"
+  cp -a yq/postgres-yq-4.* "${BOSH_INSTALL_TARGET}/bin/yq"
+  chmod +x "${BOSH_INSTALL_TARGET}/bin/yq"
+
+}
+
+main

packages/postgres-yq-4/spec

@@ -0,0 +1,4 @@
+---
+name: postgres-yq-4
+files:
+  - yq/postgres-yq-4.*

test/test_major_or_minor_version.sh

@@ -5,10 +5,18 @@ pgversion_current=$2
 # From postgres-x.y.z, it's major if x and y are not the same
 # in $pgversion_current and $pgversion_upgrade_from
 
+function check_postgresql_versions(){
+    if [[ $(echo -e "$pgversion_upgrade_from\n$pgversion_current" | sort --version-sort | head --lines=1) != $pgversion_upgrade_from ]]; then
+        echo "The downgrade of the database instance is not supported."
+        exit 1
+    fi
+}
+
 function is_major() {
   [ "${pgversion_current%.*}" != "${pgversion_upgrade_from%.*}" ]
 }
-if is_major; then
+
+if is_major && check_postgresql_versions; then
  echo is major
 else
  echo is minor