Merge pull request #15155 from Budibase/fix/pg-connection-error

Fix for fetching schemas in Postgres 9.5

.github/workflows/budibase_ci.yml

@@ -155,7 +155,7 @@ jobs:
     strategy:
       matrix:
         datasource:
-          [mssql, mysql, postgres, mongodb, mariadb, oracle, sqs, none]
+          [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -190,6 +190,8 @@ jobs:
             docker pull mariadb@${{ steps.dotenv.outputs.MARIADB_SHA }}
           elif [ "${{ matrix.datasource }}" == "oracle" ]; then
             docker pull budibase/oracle-database:23.2-slim-faststart
+          elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then
+            docker pull postgres:9.5.25
           fi
           docker pull minio/minio &
           docker pull redis &

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -596,7 +596,7 @@ const datasources = datasourceDescribe({
 if (datasources.length) {
   describe.each(datasources)(
     "$dbName",
-    ({ config, dsProvider, isPostgres, isMySQL, isMariaDB }) => {
+    ({ config, dsProvider, isPostgres, isLegacy, isMySQL, isMariaDB }) => {
       let datasource: Datasource
       let client: Knex
 
@@ -647,6 +647,13 @@ if (datasources.length) {
             // can load it. We're using postgres 16 in tests at the time of writing.
             schema = schema.replace("SET transaction_timeout = 0;", "")
           }
+          if (isPostgres && isLegacy) {
+            // in older versions of Postgres, this is not a valid option - Postgres 9.5 does not support this.
+            schema = schema.replace(
+              "SET idle_in_transaction_session_timeout = 0;",
+              ""
+            )
+          }
 
           await config.api.table.destroy(table._id!, table._rev!)
 

packages/server/src/integration-test/postgres.spec.ts

@@ -1,5 +1,4 @@
 import { Datasource, FieldType, Table } from "@budibase/types"
-import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import {
   DatabaseName,
@@ -8,7 +7,9 @@ import {
 } from "../integrations/tests/utils"
 import { Knex } from "knex"
 
-const mainDescriptions = datasourceDescribe({ only: [DatabaseName.POSTGRES] })
+const mainDescriptions = datasourceDescribe({
+  only: [DatabaseName.POSTGRES, DatabaseName.POSTGRES_LEGACY],
+})
 
 if (mainDescriptions.length) {
   describe.each(mainDescriptions)(

packages/server/src/integrations/postgres.ts

@@ -173,8 +173,13 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   `
 
   COLUMNS_SQL = () => `
-    select * from information_schema.columns where table_schema = ANY(current_schemas(false)) 
-      AND pg_table_is_visible(to_regclass(format('%I.%I', table_schema, table_name)));
+  SELECT columns.*
+  FROM information_schema.columns columns
+  JOIN pg_class pg_class ON pg_class.relname = columns.table_name
+  JOIN pg_namespace name_space ON name_space.oid = pg_class.relnamespace
+  WHERE columns.table_schema = ANY(current_schemas(false))
+    AND columns.table_schema = name_space.nspname
+    AND pg_table_is_visible(pg_class.oid);
   `
 
   constructor(config: PostgresConfig) {

packages/server/src/integrations/tests/utils/images.ts

@@ -9,5 +9,6 @@ dotenv.config({
 export const MSSQL_IMAGE = `mcr.microsoft.com/mssql/server@${process.env.MSSQL_SHA}`
 export const MYSQL_IMAGE = `mysql@${process.env.MYSQL_SHA}`
 export const POSTGRES_IMAGE = `postgres@${process.env.POSTGRES_SHA}`
+export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25`
 export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}`
 export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}`

packages/server/src/integrations/tests/utils/index.ts

@@ -16,6 +16,7 @@ export const { startContainer } = testContainerUtils
 
 export enum DatabaseName {
   POSTGRES = "postgres",
+  POSTGRES_LEGACY = "postgres_legacy",
   MONGODB = "mongodb",
   MYSQL = "mysql",
   SQL_SERVER = "mssql",
@@ -26,6 +27,7 @@ export enum DatabaseName {
 
 const providers: Record<DatabaseName, DatasourceProvider> = {
   [DatabaseName.POSTGRES]: postgres.getDatasource,
+  [DatabaseName.POSTGRES_LEGACY]: postgres.getLegacyDatasource,
   [DatabaseName.MONGODB]: mongodb.getDatasource,
   [DatabaseName.MYSQL]: mysql.getDatasource,
   [DatabaseName.SQL_SERVER]: mssql.getDatasource,
@@ -145,7 +147,11 @@ export function datasourceDescribe(opts: DatasourceDescribeOpts) {
       DatabaseName.ORACLE,
     ].includes(dbName),
     isMySQL: dbName === DatabaseName.MYSQL,
-    isPostgres: dbName === DatabaseName.POSTGRES,
+    isPostgres:
+      dbName === DatabaseName.POSTGRES ||
+      dbName === DatabaseName.POSTGRES_LEGACY,
+    // check if any of the legacy tags
+    isLegacy: dbName === DatabaseName.POSTGRES_LEGACY,
     isMongodb: dbName === DatabaseName.MONGODB,
     isMSSQL: dbName === DatabaseName.SQL_SERVER,
     isOracle: dbName === DatabaseName.ORACLE,

packages/server/src/integrations/tests/utils/postgres.ts

@@ -3,14 +3,14 @@ import { GenericContainer, Wait } from "testcontainers"
 import { generator, testContainerUtils } from "@budibase/backend-core/tests"
 import { startContainer } from "."
 import knex, { Knex } from "knex"
-import { POSTGRES_IMAGE } from "./images"
+import { POSTGRES_IMAGE, POSTGRES_LEGACY_IMAGE } from "./images"
 
 let ports: Promise<testContainerUtils.Port[]>
 
-export async function getDatasource(): Promise<Datasource> {
+async function datasourceWithImage(image: string): Promise<Datasource> {
   if (!ports) {
     ports = startContainer(
-      new GenericContainer(POSTGRES_IMAGE)
+      new GenericContainer(image)
         .withExposedPorts(5432)
         .withEnvironment({ POSTGRES_PASSWORD: "password" })
         .withWaitStrategy(
@@ -51,6 +51,14 @@ export async function getDatasource(): Promise<Datasource> {
   return datasource
 }
 
+export async function getDatasource(): Promise<Datasource> {
+  return datasourceWithImage(POSTGRES_IMAGE)
+}
+
+export async function getLegacyDatasource(): Promise<Datasource> {
+  return datasourceWithImage(POSTGRES_LEGACY_IMAGE)
+}
+
 export async function knexClient(
   ds: Datasource,
   opts?: Knex.Config