Add db-prepared-statements config

CHANGELOG.md

@@ -16,9 +16,10 @@ This project adheres to [Semantic Versioning](http://semver.org/).
  - #504, Add `log-level` config option. The admitted levels are: crit, error, warn and info - @steve-chavez
  - #1607, Enable embedding through multiple views recursively - @wolfgangwalther
  - #1598, Allow rollback of the transaction with Prefer tx=rollback - @wolfgangwalther
+ - #1633, Enable prepared statements for filters. When behind a connection pooler, you can disable preparing with `db-prepared-statements=false` - @steve-chavez
 
 ### Fixed
- 
+
  - #1592, Removed single column restriction to allow composite foreign keys in join tables - @goteguru
  - #1530, Fix how the PostgREST version is shown in the help text when the `.git` directory is not available - @monacoremo
  - #1094, Fix expired JWTs starting an empty transaction on the db - @steve-chavez

main/Main.hs

@@ -244,7 +244,7 @@ connectionStatus pool =
 fillSchemaCache :: P.Pool -> PgVersion -> IORef AppConfig -> IORef (Maybe DbStructure) -> IO ()
 fillSchemaCache pool actualPgVersion refConf refDbStructure = do
   conf <- readIORef refConf
-  result <- P.use pool $ HT.transaction HT.ReadCommitted HT.Read $ getDbStructure (toList $ configSchemas conf) (configExtraSearchPath conf) actualPgVersion
+  result <- P.use pool $ HT.transaction HT.ReadCommitted HT.Read $ getDbStructure (toList $ configSchemas conf) (configExtraSearchPath conf) actualPgVersion (configDbPrepared conf)
   case result of
     Left e -> do
       -- If this error happens it would mean the connection is down again. Improbable because connectionStatus ensured the connection.

src/PostgREST/App.hs

@@ -129,8 +129,8 @@ app dbStructure conf apiRequest =
                              then limitedQuery cq ((+ 1) <$> maxRows) -- LIMIT maxRows + 1 so we can determine below that maxRows was surpassed
                              else cq
                   stm = createReadStatement q cQuery (contentType == CTSingularJSON) shouldCount
-                        (contentType == CTTextCSV) bField pgVer
-                  explStm = createExplainStatement cq
+                        (contentType == CTTextCSV) bField pgVer prepared
+                  explStm = createExplainStatement cq prepared
               row <- H.statement mempty stm
               let (tableTotal, queryTotal, _ , body, gucHeaders, gucStatus) = row
                   gucs =  (,) <$> gucHeaders <*> gucStatus
@@ -162,7 +162,7 @@ app dbStructure conf apiRequest =
               let pkCols = tablePKCols dbStructure tSchema tName
                   stm = createWriteStatement sq mq
                     (contentType == CTSingularJSON) True
-                    (contentType == CTTextCSV) (iPreferRepresentation apiRequest) pkCols pgVer
+                    (contentType == CTTextCSV) (iPreferRepresentation apiRequest) pkCols pgVer prepared
               row <- H.statement mempty stm
               let (_, queryTotal, fields, body, gucHeaders, gucStatus) = row
                   gucs =  (,) <$> gucHeaders <*> gucStatus
@@ -197,7 +197,7 @@ app dbStructure conf apiRequest =
               row <- H.statement mempty $
                      createWriteStatement sq mq
                        (contentType == CTSingularJSON) False (contentType == CTTextCSV)
-                       (iPreferRepresentation apiRequest) [] pgVer
+                       (iPreferRepresentation apiRequest) [] pgVer prepared
               let (_, queryTotal, _, body, gucHeaders, gucStatus) = row
                   gucs =  (,) <$> gucHeaders <*> gucStatus
               case gucs of
@@ -230,7 +230,7 @@ app dbStructure conf apiRequest =
               else do
                 row <- H.statement mempty $
                        createWriteStatement sq mq (contentType == CTSingularJSON) False
-                                            (contentType == CTTextCSV) (iPreferRepresentation apiRequest) [] pgVer
+                                            (contentType == CTTextCSV) (iPreferRepresentation apiRequest) [] pgVer prepared
                 let (_, queryTotal, _, body, gucHeaders, gucStatus) = row
                     gucs =  (,) <$> gucHeaders <*> gucStatus
                 case gucs of
@@ -256,7 +256,7 @@ app dbStructure conf apiRequest =
               let stm = createWriteStatement sq mq
                     (contentType == CTSingularJSON) False
                     (contentType == CTTextCSV)
-                    (iPreferRepresentation apiRequest) [] pgVer
+                    (iPreferRepresentation apiRequest) [] pgVer prepared
               row <- H.statement mempty stm
               let (_, queryTotal, _, body, gucHeaders, gucStatus) = row
                   gucs =  (,) <$> gucHeaders <*> gucStatus
@@ -295,10 +295,11 @@ app dbStructure conf apiRequest =
             Right (q, cq, bField, returning) -> do
               let
                 preferParams = iPreferParameters apiRequest
-                pq = requestToCallProcQuery (QualifiedIdentifier pdSchema pdName) (specifiedProcArgs (iColumns apiRequest) proc) (iPayload apiRequest) returnsScalar preferParams returning
+                pq = requestToCallProcQuery (QualifiedIdentifier pdSchema pdName) (specifiedProcArgs (iColumns apiRequest) proc)
+                       (iPayload apiRequest) returnsScalar preferParams returning
                 stm = callProcStatement returnsScalar pq q cq shouldCount (contentType == CTSingularJSON)
                         (contentType == CTTextCSV) (contentType `elem` rawContentTypes) (preferParams == Just MultipleObjects)
-                        bField pgVer
+                        bField pgVer prepared
               row <- H.statement mempty stm
               let (tableTotal, queryTotal, body, gucHeaders, gucStatus) = row
                   gucs =  (,) <$> gucHeaders <*> gucStatus
@@ -340,6 +341,7 @@ app dbStructure conf apiRequest =
       where
         notFound = responseLBS status404 [] ""
         maxRows = configMaxRows conf
+        prepared = configDbPrepared conf
         exactCount = iPreferCount apiRequest == Just ExactCount
         estimatedCount = iPreferCount apiRequest == Just EstimatedCount
         plannedCount = iPreferCount apiRequest == Just PlannedCount

src/PostgREST/Config.hs

@@ -100,6 +100,8 @@ data AppConfig = AppConfig {
 
   , configTxRollbackAll     :: Bool
   , configTxAllowOverride   :: Bool
+
+  , configDbPrepared        :: Bool
   }
 
 configPoolTimeout' :: (Fractional a) => AppConfig -> a
@@ -179,6 +181,10 @@ readPathShowHelp = customExecParser parserPrefs opts
           |##   transaction is rolled back, but can be overriden with Prefer tx=commit header
           |db-tx-end = "commit"
           |
+          |## enable or disable prepared statements. disabling is only necessary when behind a connection pooler.
+          |## when disabled, statements will be parametrized but won't be prepared.
+          |db-prepared-statements = true
+          |
           |server-host = "!4"
           |server-port = 3000
           |
@@ -263,6 +269,7 @@ readAppConfig cfgPath = do
         <*> parseLogLevel "log-level"
         <*> parseTxEnd "db-tx-end" fst
         <*> parseTxEnd "db-tx-end" snd
+        <*> (fromMaybe True <$>  optBool "db-prepared-statements")
 
     parseSocketFileMode :: C.Key -> C.Parser C.Config (Either Text FileMode)
     parseSocketFileMode k =

src/PostgREST/DbStructure.hs

@@ -44,15 +44,15 @@ import Text.InterpolatedString.Perl6 (q, qc)
 import PostgREST.Private.Common
 import PostgREST.Types
 
-getDbStructure :: [Schema] -> [Schema] -> PgVersion -> HT.Transaction DbStructure
-getDbStructure schemas extraSearchPath pgVer = do
+getDbStructure :: [Schema] -> [Schema] -> PgVersion -> Bool -> HT.Transaction DbStructure
+getDbStructure schemas extraSearchPath pgVer prepared = do
   HT.sql "set local schema ''" -- This voids the search path. The following queries need this for getting the fully qualified name(schema.name) of every db object
-  tabs    <- HT.statement () allTables
-  cols    <- HT.statement schemas $ allColumns tabs
-  srcCols <- HT.statement (schemas, extraSearchPath) $ pfkSourceColumns cols pgVer
-  m2oRels <- HT.statement () $ allM2ORels tabs cols
-  keys    <- HT.statement () $ allPrimaryKeys tabs
-  procs   <- HT.statement schemas allProcs
+  tabs    <- HT.statement () $ allTables prepared
+  cols    <- HT.statement schemas $ allColumns tabs prepared
+  srcCols <- HT.statement (schemas, extraSearchPath) $ pfkSourceColumns cols pgVer prepared
+  m2oRels <- HT.statement () $ allM2ORels tabs cols prepared
+  keys    <- HT.statement () $ allPrimaryKeys tabs prepared
+  procs   <- HT.statement schemas $ allProcs prepared
 
   let rels = addM2MRels . addO2MRels $ addViewM2ORels srcCols m2oRels
       cols' = addForeignKeys rels cols
@@ -183,8 +183,8 @@ decodeProcs =
                       | v == 's' = Stable
                       | otherwise = Volatile -- only 'v' can happen here
 
-allProcs :: H.Statement [Schema] ProcsMap
-allProcs = H.Statement (toS sql) (arrayParam HE.text) decodeProcs True
+allProcs :: Bool -> H.Statement [Schema] ProcsMap
+allProcs = H.Statement (toS sql) (arrayParam HE.text) decodeProcs
   where
     sql = procsSqlQuery <> " WHERE pn.nspname = ANY($1)"
 
@@ -407,9 +407,9 @@ addViewPrimaryKeys srcCols = concatMap (\pk ->
                 filter (\(col, _) -> colTable col == pkTable pk && colName col == pkName pk) srcCols in
   pk : viewPks)
 
-allTables :: H.Statement () [Table]
+allTables :: Bool -> H.Statement () [Table]
 allTables =
-  H.Statement sql HE.noParams decodeTables True
+  H.Statement sql HE.noParams decodeTables
  where
   sql = [q|
     SELECT
@@ -434,9 +434,9 @@ allTables =
     GROUP BY table_schema, table_name, insertable
     ORDER BY table_schema, table_name |]
 
-allColumns :: [Table] -> H.Statement [Schema] [Column]
+allColumns :: [Table] -> Bool -> H.Statement [Schema] [Column]
 allColumns tabs =
- H.Statement sql (arrayParam HE.text) (decodeColumns tabs) True
+ H.Statement sql (arrayParam HE.text) (decodeColumns tabs)
  where
   sql = [q|
     SELECT DISTINCT
@@ -580,9 +580,9 @@ columnFromRow tabs (s, t, n, desc, pos, nul, typ, u, l, p, d, e) = buildColumn <
     parseEnum :: Maybe Text -> [Text]
     parseEnum = maybe [] (split (==','))
 
-allM2ORels :: [Table] -> [Column] -> H.Statement () [Relation]
+allM2ORels :: [Table] -> [Column] -> Bool -> H.Statement () [Relation]
 allM2ORels tabs cols =
-  H.Statement sql HE.noParams (decodeRels tabs cols) True
+  H.Statement sql HE.noParams (decodeRels tabs cols)
  where
   sql = [q|
     SELECT ns1.nspname AS table_schema,
@@ -618,9 +618,9 @@ relFromRow allTabs allCols (rs, rt, cn, rcs, frs, frt, frcs) =
     cols  = mapM (findCol rs rt) rcs
     colsF = mapM (findCol frs frt) frcs
 
-allPrimaryKeys :: [Table] -> H.Statement () [PrimaryKey]
+allPrimaryKeys :: [Table] -> Bool -> H.Statement () [PrimaryKey]
 allPrimaryKeys tabs =
-  H.Statement sql HE.noParams (decodePks tabs) True
+  H.Statement sql HE.noParams (decodePks tabs)
  where
   sql = [q|
     -- CTE to replace information_schema.table_constraints to remove owner limit
@@ -699,9 +699,9 @@ pkFromRow tabs (s, t, n) = PrimaryKey <$> table <*> pure n
   where table = find (\tbl -> tableSchema tbl == s && tableName tbl == t) tabs
 
 -- returns all the primary and foreign key columns which are referenced in views
-pfkSourceColumns :: [Column] -> PgVersion -> H.Statement ([Schema], [Schema]) [SourceColumn]
+pfkSourceColumns :: [Column] -> PgVersion -> Bool -> H.Statement ([Schema], [Schema]) [SourceColumn]
 pfkSourceColumns cols pgVer =
-  H.Statement sql (contrazip2 (arrayParam HE.text) (arrayParam HE.text)) (decodeSourceColumns cols) True
+  H.Statement sql (contrazip2 (arrayParam HE.text) (arrayParam HE.text)) (decodeSourceColumns cols)
   -- query explanation at https://gist.github.com/steve-chavez/7ee0e6590cddafb532e5f00c46275569
   where
     subselectRegex :: Text

src/PostgREST/Statements.hs

@@ -44,10 +44,10 @@ import qualified Hasql.DynamicStatements.Statement as H
 type ResultsWithCount = (Maybe Int64, Int64, [BS.ByteString], BS.ByteString, Either SimpleError [GucHeader], Either SimpleError (Maybe Status))
 
 createWriteStatement :: H.Snippet -> H.Snippet -> Bool -> Bool -> Bool ->
-                        PreferRepresentation -> [Text] -> PgVersion ->
+                        PreferRepresentation -> [Text] -> PgVersion -> Bool ->
                         H.Statement () ResultsWithCount
 createWriteStatement selectQuery mutateQuery wantSingle isInsert asCsv rep pKeys pgVer =
-  H.dynamicallyParameterized snippet decodeStandard True
+  H.dynamicallyParameterized snippet decodeStandard
  where
   snippet =
     "WITH " <> H.sql sourceCTEName <> " AS (" <> mutateQuery <> ") " <>
@@ -86,10 +86,10 @@ createWriteStatement selectQuery mutateQuery wantSingle isInsert asCsv rep pKeys
   decodeStandard =
    fromMaybe (Nothing, 0, [], mempty, Right [], Right Nothing) <$> HD.rowMaybe standardRow
 
-createReadStatement :: H.Snippet -> H.Snippet -> Bool -> Bool -> Bool -> Maybe FieldName -> PgVersion ->
+createReadStatement :: H.Snippet -> H.Snippet -> Bool -> Bool -> Bool -> Maybe FieldName -> PgVersion -> Bool ->
                        H.Statement () ResultsWithCount
 createReadStatement selectQuery countQuery isSingle countTotal asCsv binaryField pgVer =
-  H.dynamicallyParameterized snippet decodeStandard True
+  H.dynamicallyParameterized snippet decodeStandard
  where
   snippet =
     "WITH " <>
@@ -129,10 +129,10 @@ standardRow = (,,,,,) <$> nullableColumn HD.int8 <*> column HD.int8
 type ProcResults = (Maybe Int64, Int64, ByteString, Either SimpleError [GucHeader], Either SimpleError (Maybe Status))
 
 callProcStatement :: Bool -> H.Snippet -> H.Snippet -> H.Snippet -> Bool ->
-                     Bool -> Bool -> Bool -> Bool -> Maybe FieldName -> PgVersion ->
+                     Bool -> Bool -> Bool -> Bool -> Maybe FieldName -> PgVersion -> Bool ->
                      H.Statement () ProcResults
 callProcStatement returnsScalar callProcQuery selectQuery countQuery countTotal isSingle asCsv asBinary multObjects binaryField pgVer =
-  H.dynamicallyParameterized snippet decodeProc True
+  H.dynamicallyParameterized snippet decodeProc
   where
     snippet =
       "WITH " <> H.sql sourceCTEName <> " AS (" <> callProcQuery <> ") " <>
@@ -171,9 +171,9 @@ callProcStatement returnsScalar callProcQuery selectQuery countQuery countTotal
                          <*> (fromMaybe defGucHeaders <$> nullableColumn decodeGucHeaders)
                          <*> (fromMaybe defGucStatus <$> nullableColumn decodeGucStatus)
 
-createExplainStatement :: H.Snippet -> H.Statement () (Maybe Int64)
+createExplainStatement :: H.Snippet -> Bool -> H.Statement () (Maybe Int64)
 createExplainStatement countQuery =
-  H.dynamicallyParameterized snippet decodeExplain True
+  H.dynamicallyParameterized snippet decodeExplain
   where
     snippet = "EXPLAIN (FORMAT JSON) " <> countQuery
     -- |

test/Main.hs

@@ -200,4 +200,4 @@ main = do
 
   where
     setupDbStructure pool schemas extraSearchPath ver =
-      either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ getDbStructure (toList schemas) extraSearchPath ver)
+      either (panic.show) id <$> P.use pool (HT.transaction HT.ReadCommitted HT.Read $ getDbStructure (toList schemas) extraSearchPath ver True)

test/SpecHelper.hs

@@ -92,6 +92,7 @@ _baseCfg = let secret = Just $ encodeUtf8 "reallyreallyreallyreallyverysafe" in
   , configLogLevel          = LogCrit
   , configTxRollbackAll     = False
   , configTxAllowOverride   = True
+  , configDbPrepared        = True
   }
 
 testCfg :: Text -> AppConfig

test/fixtures/schema.sql

@@ -1149,7 +1149,7 @@ $$;
 
 CREATE FUNCTION test.sayhello_variadic(name TEXT, VARIADIC v TEXT[]) RETURNS text
 LANGUAGE SQL AS $$
-  SELECT 'Hello, ' || name 
+  SELECT 'Hello, ' || name
 $$;
 
 create or replace function test.raise_pt402() returns void as $$
@@ -1859,3 +1859,7 @@ create function v2.get_parents_below(id int)
 returns setof v2.parents as $$
   select * from v2.parents where id < $1;
 $$ language sql;
+
+-- Only used for manually testing creating prepared statements
+create view prepared_statements as
+select * from pg_catalog.pg_prepared_statements;