feat: Add the possibility to specify the used PostgreSQL version

README.md

@@ -76,6 +76,7 @@ The complete list of available properties can be found in the [spec](jobs/postgr
 
 Property | Description
 -------- | -------------
+databases.version | Define the used PostgreSQL major version. Default: 15
 databases.port | The database port. Default: 5432
 databases.databases | A list of databases and associated properties to create when Postgres starts
 databases.databases[n].name | Database name

ci/pipeline.yml

@@ -86,6 +86,18 @@ jobs:
             options:
               access_key_id: ((postgres-release-blobstore-user.username))
               secret_access_key: ((postgres-release-blobstore-user.password))
+    - task: bump-yq-package
+      file: postgres-release/ci/tasks/bump-yq-packages/task.yml
+      image: bosh-cli-registry-image
+      input_mapping:
+        yq-src: yq-4-src
+      params:
+        MAJOR_VERSION: 1
+        PRIVATE_YML: |
+          blobstore:
+            options:
+              access_key_id: ((postgres-release-blobstore-user.username))
+              secret_access_key: ((postgres-release-blobstore-user.password))
     - put: postgres-release
       params:
         repository: postgres-release
@@ -232,6 +244,13 @@ resources:
       regex: 'href="v(?P<version>16\.[0-9.]+)/"'
       uri: "https://ftp.postgresql.org/pub/source/v{version}/postgresql-{version}.tar.gz"
 
+  - name: yq-4-src
+    type: http-resource
+    source:
+      index: "https://api.github.com/repos/mikefarah/yq/releases"
+      regex: '"tag_name":"v(?P<version>4\.[0-9.]+)/"'
+      uri: "https://github.com/mikefarah/yq/releases/download/v{version}/yq_linux_amd64"
+
   - name: postgres-release
     type: git
     source:
@@ -263,4 +282,3 @@ resource_types:
     type: docker-image
     source:
       repository: aequitas/http-resource
-

ci/tasks/bump-postgres-packages/task.sh

@@ -19,14 +19,9 @@ pushd postgres-release
   fi
 
   if ${NEED_COMMIT}; then
-    eval $(grep "^current_version=" jobs/postgres/templates/pgconfig.sh.erb)
-    new_version=$(echo "${FILENAME}" | egrep -o "[0-9.]+[0-9]")
-
-    # If new version is greater than current version, update the current_version variables
-    if printf '%s\n' "$current_version" "$new_version" | sort -V -C; then
-      sed -i "s/^current_version=.*/current_version=\"${new_version}\"/" jobs/postgres/templates/pgconfig.sh.erb
-      sed -i "s/^current_version=.*/current_version=\"${new_version}\"/" jobs/bbr-postgres-db/templates/config.sh.erb
-    fi
+    echo "-----> $(date): Update the PostgreSQL version inside the used_postgresql_versions.yml file"
+    current_minor_version=$(cat config/blobs.yml  | grep "postgresql-${MAJOR_VERSION}" | cut -f1 -d: | sed "s/postgres\/postgresql-//g" | sed "s/.tar.gz//g")
+    CURRENT_MINOR_VERSION=$current_minor_version /tmp/yq -i '.postgresql.major_version[env(MAJOR_VERSION)].minor_version = strenv(CURRENT_MINOR_VERSION)' jobs/postgres/config/used_postgresql_versions.yml
 
     echo "-----> $(date): Creating git commit"
     git config user.name "$GIT_USER_NAME"

ci/tasks/bump-yq-packages/task.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+NEED_COMMIT=false
+
+set -euo pipefail
+
+echo "${PRIVATE_YML}" > postgres-release/config/private.yml
+
+pushd postgres-release
+  CURRENT_BLOBS=$(bosh blobs)
+  CURRENT_VERSION=$(curl -s -L https://api.github.com/repos/mikefarah/yq/releases/latest | grep "tag_name" | sed s/\"tag_name\":\//g | sed s/\"//g | sed s/\,//g | sed s/v//g | xargs)
+  mv ../yq-src/yq_darwin_amd64 ../yq-src/yq-${CURRENT_VERSION}
+  BLOB_PATH=$(ls ../yq-src/yq-${CURRENT_VERSION})
+  FILENAME=$( basename ${BLOB_PATH} )
+  OLD_BLOB_PATH=$(cat config/blobs.yml  | grep "yq-${MAJOR_VERSION}" | cut -f1 -d:)
+  if ! echo "${CURRENT_BLOBS}" | grep "${FILENAME}" ; then
+    NEED_COMMIT=true
+    echo "adding ${FILENAME}"
+    bosh add-blob --sha2 "${BLOB_PATH}" "yq/${FILENAME}"
+    bosh remove-blob ${OLD_BLOB_PATH}
+    bosh upload-blobs
+  fi
+
+  if ${NEED_COMMIT}; then
+    echo "-----> $(date): Creating git commit"
+    git config user.name "$GIT_USER_NAME"
+    git config user.email "$GIT_USER_EMAIL"
+    git add .
+
+    git --no-pager diff --cached
+    if [[ "$( git status --porcelain )" != "" ]]; then
+      git commit -am "Bump packages"
+    fi
+  fi
+popd

ci/tasks/bump-yq-packages/task.yml

@@ -0,0 +1,17 @@
+---
+platform: linux
+
+inputs:
+- name: yq-src
+
+outputs:
+- name: postgres-release
+
+run:
+  path: postgres-release/ci/tasks/bump-yq-packages/task.sh
+
+params:
+  PRIVATE_YML:
+  MAJOR_VERSION:
+  GIT_USER_NAME: CI Bot
+  GIT_USER_EMAIL: bots@cloudfoundry.org

ci/tasks/create-final-release/task.sh

@@ -8,11 +8,15 @@ set +x
 echo "$RELEASE_PRIVATE_YML" >> "config/private.yml"
 set -x
 
+latest_yq_version=$(curl -s -L https://api.github.com/repos/mikefarah/yq/releases/latest | grep "tag_name" | sed s/\"tag_name\":\//g | sed s/\"//g | sed s/\,//g | sed s/v//g | xargs)
+curl -s -L https://github.com/mikefarah/yq/releases/download/v${latest_yq_version}/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq
+
 bosh -n create-release --final
 new_release_version="$(find releases -regex ".*postgres-[0-9]*.yml" | egrep -o "[0-9]+" | sort -n | tail -n 1)"
 
-eval $(grep "^current_version=" jobs/postgres/templates/pgconfig.sh.erb)
-sed -i "/^versions:/a\ \ ${new_release_version}: \"PostgreSQL ${current_version}\"" versions.yml
+current_major_version=$(/tmp/yq '.postgresql.default' jobs/postgres/config/used_postgresql_versions.yml)
+current_minor_version=$(CURRENT_MAJOR_VERSION=$current_major_version /tmp/yq '.postgresql.major_version[env(CURRENT_MAJOR_VERSION)].minor_version' jobs/postgres/config/used_postgresql_versions.yml)
+sed -i "/^versions:/a\ \ ${new_release_version}: \"PostgreSQL ${current_minor_version}\"" versions.yml
 git add versions.yml
 
 git add .final_builds releases

ci/tasks/run-acceptance-tests/task.sh

@@ -2,6 +2,11 @@
 
 root="${PWD}"
 
+function setup_yq() {
+    latest_yq_version=$(curl -s -L https://api.github.com/repos/mikefarah/yq/releases/latest | grep "tag_name" | sed s/\"tag_name\":\//g | sed s/\"//g | sed s/\,//g | sed s/v//g | xargs)
+    curl -s -L https://github.com/mikefarah/yq/releases/download/v${latest_yq_version}/yq_linux_amd64 -o /tmp/yq && chmod +x /tmp/yq
+}
+
 function setup_bosh() {
   source start-bosh
   source /tmp/local-bosh/director/env
@@ -43,10 +48,13 @@ function upload_release() {
 
 function main() {
   setup_bosh
+  setup_yq
   install_bbr
   upload_release
   bosh upload-stemcell stemcell/stemcell.tgz
-  cat ${root}/postgres-release/jobs/postgres/templates/pgconfig.sh.erb | grep current_version > ${root}/pgconfig.sh
+  current_major_version=$(/tmp/yq '.postgresql.default' jobs/postgres/config/used_postgresql_versions.yml)
+  current_minor_version=$(CURRENT_MAJOR_VERSION=$current_major_version /tmp/yq '.postgresql.major_version[env(CURRENT_MAJOR_VERSION)].minor_version' jobs/postgres/config/used_postgresql_versions.yml)
+  echo "current_version=${current_minor_version}" > ${root}/pgconfig.sh
   source ${root}/pgconfig.sh
   config_file="${root}/pgats_config.yml"
   create_config_file > $config_file

jobs/bbr-postgres-db/spec

@@ -18,6 +18,7 @@ packages:
   - postgres-13
   - postgres-15
   - postgres-16
+  - yq-4
 
 consumes:
 - name: database

jobs/bbr-postgres-db/templates/config.sh.erb

@@ -22,9 +22,10 @@ if_link("database") do |data|
 end
 
  %>
-current_version="16.3"
+
+current_version=postgres-$(MAJOR_VERSION=<%=p('databases.version')%> ${JOB_DIR}/packages/yq-4/bin/yq '.postgresql.major_version[env(MAJOR_VERSION)].minor_version' /var/vcap/jobs/postgres/config/used_postgresql_versions.yml)
 JOB_DIR="/var/vcap/jobs/bbr-postgres-db"
-PACKAGE_DIR="/var/vcap/packages/postgres-${current_version%.*}"
+PACKAGE_DIR="/var/vcap/packages/${current_version%.*}"
 PORT="<%= port %>"
 DATABASES=(<%= databases.map{|d| d["name"]}.join(' ')%>)
 DBHOST=<%= dbhost %>

jobs/postgres/spec

@@ -24,13 +24,15 @@ templates:
   hooks/postgres-pre-stop.sh.erb: bin/hooks/postgres-pre-stop.sh
   hooks/postgres-post-start.sh.erb: bin/hooks/postgres-post-start.sh
   hooks/postgres-post-stop.sh.erb: bin/hooks/postgres-post-stop.sh
+  used_postgresql_versions.yml: config/used_postgresql_versions.yml
 
 packages:
   - postgres-common
   - postgres-11
   - postgres-13
   - postgres-15
   - postgres-16
+  - yq-4
 
 provides:
 - name: postgres
@@ -42,6 +44,9 @@ provides:
   - databases.tls.ca
 
 properties:
+  databases.version:
+      description: "The database version e.g. 11, 13, 15 or 16"
+      default: "16"
   databases.port:
     description: "The database port"
     default: 5432

jobs/postgres/templates/pgconfig.sh.erb

@@ -6,10 +6,10 @@ set -x # if you want tracing disabled, set 'databases.enable_traces: false' in t
 ENABLE_TRACE=0
 # set -x # uncomment it if you want to enable tracing in all control scripts
 <% end %>
-current_version="16.3"
-pgversion_current="postgres-${current_version}"
-
 JOB_DIR=/var/vcap/jobs/postgres
+current_version="<%=p('databases.version')%>"
+pgversion_current=postgres-$(MAJOR_VERSION=<%=p('databases.version')%> ${JOB_DIR}/packages/yq-4/bin/yq '.postgresql.major_version[env(MAJOR_VERSION)].minor_version' /var/vcap/jobs/postgres/config/used_postgresql_versions.yml)
+
 PACKAGE_DIR="/var/vcap/packages/${pgversion_current%.*}"
 STORE_DIR=/var/vcap/store
 PG_STORE_DIR=${STORE_DIR}/postgres

jobs/postgres/templates/used_postgresql_versions.yml

@@ -0,0 +1,11 @@
+postgresql:
+  default: 16
+  major_version:
+    16:
+      minor_version: "16.3"
+    15:
+      minor_version: "15.6"
+    13:
+      minor_version: "13.14"
+    11:
+      minor_version: "11.22"

packages/yq-4/packaging

@@ -0,0 +1,17 @@
+#!/bin/bash -exu
+
+function main() {
+  install
+
+}
+
+function install() {
+
+  echo "Install yq..."
+  mkdir -p "${BOSH_INSTALL_TARGET}/bin"
+  cp -a yq/yq-4.* "${BOSH_INSTALL_TARGET}/bin/yq"
+  chmod +x "${BOSH_INSTALL_TARGET}/bin/yq"
+
+}
+
+main

packages/yq-4/spec

@@ -0,0 +1,4 @@
+---
+name: yq-4
+files:
+  - yq/yq-4.*