Merge branch 'master' into custom-column-names-update

.circleci/config.yml

@@ -118,8 +118,9 @@ refs:
     - run:
         name: Run Python tests
         environment:
-          # hpc report seems to fail with the default -N
-          GHCRTS: -N1
+          # Setting default number of threads to 2
+          # since circleci allocates 2 cpus per test container
+          GHCRTS: -N2
           HASURA_GRAPHQL_DATABASE_URL: 'postgres://gql_test:@localhost:5432/gql_test'
           HASURA_GRAPHQL_DATABASE_URL_2: 'postgres://gql_test:@localhost:5432/gql_test2'
           GRAPHQL_ENGINE: '/build/_server_output/graphql-engine'

.circleci/test-server.sh

@@ -86,7 +86,12 @@ combine_all_hpc_reports() {
 			continue
 		fi
 		if [ -f "$combined_file" ]  ; then
+			GHCRTS_PREV="$GHCRTS"
+			# Unsetting GHCRTS as hpc combine fails if GCHRTS=-N2 is present
+			unset GHCRTS
 			(set -x && stack --allow-different-user exec -- hpc combine "$combined_file" "$tix_file" --union --output="$combined_file_intermediate" && set +x && mv "$combined_file_intermediate" "$combined_file" && rm "$tix_file" ) || true
+			# Restoring GHCRTS
+			export GHCRTS="$GHCRTS_PREV"
 		else
 			mv "$tix_file" "$combined_file" || true
 		fi

console/README.md

@@ -67,7 +67,7 @@ Environment variables accepted in `server` mode:
 
 1. `PORT`: Configure the port where Hasura console will run locally.
 2. `NODE_ENV`: `development`
-3. `DATA_API_URL`: Configure it with the Hasura GraphQL Engine url. If you are running it on Heroku, your url will look like <app-name>.herokuapp.com.
+3. `DATA_API_URL`: Configure it with the Hasura GraphQL Engine url. If you are running it on Heroku, your url will look like `<app-name>.herokuapp.com`
 4. `ADMIN_SECRET`: Set admin secret if Hasura GraphQL engine is configured to run with ADMIN_SECRET.
 5. `CONSOLE_MODE`: `server`
 6. `URL_PREFIX`: `/` (forward slash)

server/graphql-engine.cabal

@@ -319,6 +319,7 @@ library
                       QuasiQuotes
                       RankNTypes
                       ScopedTypeVariables
+                      StandaloneDeriving
                       TemplateHaskell
                       TupleSections
                       TypeApplications
@@ -363,6 +364,7 @@ executable graphql-engine
                       QuasiQuotes
                       RankNTypes
                       ScopedTypeVariables
+                      StandaloneDeriving
                       TemplateHaskell
                       TupleSections
                       TypeApplications

server/src-lib/Hasura/GraphQL/Execute.hs

@@ -8,6 +8,7 @@ module Hasura.GraphQL.Execute
   , ExecPlanResolved
   , getResolvedExecPlan
   , execRemoteGQ
+  , getSubsOp
 
   , EP.PlanCache
   , EP.initPlanCache

server/src-lib/Hasura/GraphQL/Execute/LiveQuery.hs

@@ -93,6 +93,9 @@ module Hasura.GraphQL.Execute.LiveQuery
   , reuseLiveQueryPlan
   , buildLiveQueryPlan
 
+  , LiveQueryPlanExplanation
+  , explainLiveQueryPlan
+
   , LiveQueriesState
   , initLiveQueriesState
   , dumpLiveQueriesState

server/src-lib/Hasura/GraphQL/Execute/LiveQuery/Plan.hs

@@ -1,17 +1,26 @@
+{-# LANGUAGE UndecidableInstances #-}
+
 -- | Construction of multiplexed live query plans; see "Hasura.GraphQL.Execute.LiveQuery" for
 -- details.
 module Hasura.GraphQL.Execute.LiveQuery.Plan
   ( MultiplexedQuery
   , mkMultiplexedQuery
-  , unMultiplexedQuery
-  , toMultiplexedQueryVar
+  , resolveMultiplexedValue
+
+  , CohortId
+  , newCohortId
+  , CohortVariables
+  , executeMultiplexedQuery
 
   , LiveQueryPlan(..)
   , ParameterizedLiveQueryPlan(..)
   , ReusableLiveQueryPlan
   , ValidatedQueryVariables
   , buildLiveQueryPlan
   , reuseLiveQueryPlan
+
+  , LiveQueryPlanExplanation
+  , explainLiveQueryPlan
   ) where
 
 import           Hasura.Prelude
@@ -21,18 +30,25 @@ import qualified Data.Aeson.Extended                    as J
 import qualified Data.Aeson.TH                          as J
 import qualified Data.HashMap.Strict                    as Map
 import qualified Data.Text                              as T
+import qualified Data.UUID.V4                           as UUID
 import qualified Database.PG.Query                      as Q
 import qualified Language.GraphQL.Draft.Syntax          as G
 
+-- remove these when array encoding is merged
+import qualified Database.PG.Query.PTI                  as PTI
+import qualified PostgreSQL.Binary.Encoding             as PE
+
 import           Control.Lens
 import           Data.Has
+import           Data.UUID                              (UUID)
 
 import qualified Hasura.GraphQL.Resolve                 as GR
 import qualified Hasura.GraphQL.Transport.HTTP.Protocol as GH
 import qualified Hasura.GraphQL.Validate                as GV
 import qualified Hasura.SQL.DML                         as S
 
 import           Hasura.Db
+import           Hasura.EncJSON
 import           Hasura.RQL.Types
 import           Hasura.SQL.Error
 import           Hasura.SQL.Types
@@ -65,21 +81,23 @@ mkMultiplexedQuery baseQuery =
             ) _fld_resp ON ('true')
         |]
 
--- | converts the partial unresolved value containing
--- variables, session variables to an SQL expression
--- referring correctly to the values from '_subs' temporary table
--- The variables are at _subs.result_vars.variables and
--- session variables at _subs.result_vars.user
-toMultiplexedQueryVar :: (MonadState GV.ReusableVariableValues m) => GR.UnresolvedVal -> m S.SQLExp
-toMultiplexedQueryVar = \case
-  GR.UVPG annPGVal ->
+-- | Resolves an 'GR.UnresolvedVal' by converting 'GR.UVPG' values to SQL expressions that refer to
+-- the @result_vars@ input object, collecting variable values along the way.
+resolveMultiplexedValue
+  :: (MonadState (GV.ReusableVariableValues, Seq (WithScalarType PGScalarValue)) m)
+  => GR.UnresolvedVal -> m S.SQLExp
+resolveMultiplexedValue = \case
+  GR.UVPG annPGVal -> do
     let GR.AnnPGVal varM _ colVal = annPGVal
-    in case varM of
-      Just var -> do
-        modify $ Map.insert var colVal
-        pure $ fromResVars (PGTypeScalar $ pstType colVal)
-          ["variables", G.unName $ G.unVariable var]
-      Nothing -> return $ toTxtValue colVal
+    varJsonPath <- case varM of
+      Just varName -> do
+        modifying _1 $ Map.insert varName colVal
+        pure ["variables", G.unName $ G.unVariable varName]
+      Nothing -> do
+        syntheticVarIndex <- gets (length . snd)
+        modifying _2 (|> colVal)
+        pure ["synthetic", T.pack $ show syntheticVarIndex]
+    pure $ fromResVars (PGTypeScalar $ pstType colVal) varJsonPath
   GR.UVSessVar ty sessVar -> pure $ fromResVars ty ["user", T.toLower sessVar]
   GR.UVSQL sqlExp -> pure sqlExp
   where
@@ -89,6 +107,69 @@ toMultiplexedQueryVar = \case
       , S.SEArray $ map S.SELit jPath
       ]
 
+newtype CohortId = CohortId { unCohortId :: UUID }
+  deriving (Show, Eq, Hashable, J.ToJSON, Q.FromCol)
+
+newCohortId :: (MonadIO m) => m CohortId
+newCohortId = CohortId <$> liftIO UUID.nextRandom
+
+data CohortVariables
+  = CohortVariables
+  { _cvSessionVariables   :: !UserVars
+  , _cvQueryVariables     :: !ValidatedQueryVariables
+  , _cvSyntheticVariables :: !ValidatedSyntheticVariables
+  -- ^ To allow more queries to be multiplexed together, we introduce “synthetic” variables for
+  -- /all/ SQL literals in a query, even if they don’t correspond to any GraphQL variable. For
+  -- example, the query
+  --
+  -- > subscription latest_tracks($condition: tracks_bool_exp!) {
+  -- >   tracks(where: $tracks_bool_exp) {
+  -- >     id
+  -- >     title
+  -- >   }
+  -- > }
+  --
+  -- might be executed with similar values for @$condition@, such as @{"album_id": {"_eq": "1"}}@
+  -- and @{"album_id": {"_eq": "2"}}@.
+  --
+  -- Normally, we wouldn’t bother parameterizing over the @1@ and @2@ literals in the resulting
+  -- query because we can’t cache that query plan (since different @$condition@ values could lead to
+  -- different SQL). However, for live queries, we can still take advantage of the similarity
+  -- between the two queries by multiplexing them together, so we replace them with references to
+  -- synthetic variables.
+  } deriving (Show, Eq, Generic)
+instance Hashable CohortVariables
+
+instance J.ToJSON CohortVariables where
+  toJSON (CohortVariables sessionVars queryVars syntheticVars) =
+    J.object ["session" J..= sessionVars, "query" J..= queryVars, "synthetic" J..= syntheticVars]
+
+-- These types exist only to use the Postgres array encoding.
+newtype CohortIdArray = CohortIdArray { unCohortIdArray :: [CohortId] }
+  deriving (Show, Eq)
+instance Q.ToPrepArg CohortIdArray where
+  toPrepVal (CohortIdArray l) = Q.toPrepValHelper PTI.unknown encoder $ map unCohortId l
+    where
+      encoder = PE.array 2950 . PE.dimensionArray foldl' (PE.encodingArray . PE.uuid)
+newtype CohortVariablesArray = CohortVariablesArray { unCohortVariablesArray :: [CohortVariables] }
+  deriving (Show, Eq)
+instance Q.ToPrepArg CohortVariablesArray where
+  toPrepVal (CohortVariablesArray l) =
+    Q.toPrepValHelper PTI.unknown encoder (map J.toJSON l)
+    where
+      encoder = PE.array 114 . PE.dimensionArray foldl' (PE.encodingArray . PE.json_ast)
+
+executeMultiplexedQuery
+  :: (MonadTx m) => MultiplexedQuery -> [(CohortId, CohortVariables)] -> m [(CohortId, EncJSON)]
+executeMultiplexedQuery (MultiplexedQuery query) = executeQuery query
+
+-- | Internal; used by both 'executeMultiplexedQuery' and 'explainLiveQueryPlan'.
+executeQuery :: (MonadTx m, Q.FromRow a) => Q.Query -> [(CohortId, CohortVariables)] -> m [a]
+executeQuery query cohorts =
+  let (cohortIds, cohortVars) = unzip cohorts
+      preparedArgs = (CohortIdArray cohortIds, CohortVariablesArray cohortVars)
+  in liftTx $ Q.listQE defaultTxErrorHandler query preparedArgs True
+
 -- -------------------------------------------------------------------------------------------------
 -- Variable validation
 
@@ -100,21 +181,27 @@ toMultiplexedQueryVar = \case
 -- > SELECT 'v1'::t1, 'v2'::t2, ..., 'vn'::tn
 --
 -- so if any variable values are invalid, the error will be caught early.
-newtype ValidatedQueryVariables = ValidatedQueryVariables (Map.HashMap G.Variable TxtEncodedPGVal)
-  deriving (Show, Eq, Hashable, J.ToJSON)
+newtype ValidatedVariables f = ValidatedVariables (f TxtEncodedPGVal)
+deriving instance (Show (f TxtEncodedPGVal)) => Show (ValidatedVariables f)
+deriving instance (Eq (f TxtEncodedPGVal)) => Eq (ValidatedVariables f)
+deriving instance (Hashable (f TxtEncodedPGVal)) => Hashable (ValidatedVariables f)
+deriving instance (J.ToJSON (f TxtEncodedPGVal)) => J.ToJSON (ValidatedVariables f)
+
+type ValidatedQueryVariables = ValidatedVariables (Map.HashMap G.Variable)
+type ValidatedSyntheticVariables = ValidatedVariables []
 
 -- | Checks if the provided arguments are valid values for their corresponding types.
 -- Generates SQL of the format "select 'v1'::t1, 'v2'::t2 ..."
-validateQueryVariables
-  :: (MonadError QErr m, MonadIO m)
+validateVariables
+  :: (Traversable f, MonadError QErr m, MonadIO m)
   => PGExecCtx
-  -> GV.ReusableVariableValues
-  -> m ValidatedQueryVariables
-validateQueryVariables pgExecCtx annVarVals = do
-  let valSel = mkValidationSel $ Map.elems annVarVals
+  -> f (WithScalarType PGScalarValue)
+  -> m (ValidatedVariables f)
+validateVariables pgExecCtx variableValues = do
+  let valSel = mkValidationSel $ toList variableValues
   Q.Discard () <- runTx' $ liftTx $
     Q.rawQE dataExnErrHandler (Q.fromBuilder $ toSQL valSel) [] False
-  pure . ValidatedQueryVariables $ fmap (txtEncodedPGVal . pstValue) annVarVals
+  pure . ValidatedVariables $ fmap (txtEncodedPGVal . pstValue) variableValues
   where
     mkExtrs = map (flip S.Extractor Nothing . toTxtValue)
     mkValidationSel vars =
@@ -135,8 +222,7 @@ validateQueryVariables pgExecCtx annVarVals = do
 data LiveQueryPlan
   = LiveQueryPlan
   { _lqpParameterizedPlan :: !ParameterizedLiveQueryPlan
-  , _lqpSessionVariables  :: !UserVars
-  , _lqpQueryVariables    :: !ValidatedQueryVariables
+  , _lqpVariables         :: !CohortVariables
   }
 
 data ParameterizedLiveQueryPlan
@@ -149,8 +235,9 @@ $(J.deriveToJSON (J.aesonDrop 4 J.snakeCase) ''ParameterizedLiveQueryPlan)
 
 data ReusableLiveQueryPlan
   = ReusableLiveQueryPlan
-  { _rlqpParameterizedPlan  :: !ParameterizedLiveQueryPlan
-  , _rlqpQueryVariableTypes :: !GV.ReusableVariableTypes
+  { _rlqpParameterizedPlan       :: !ParameterizedLiveQueryPlan
+  , _rlqpSyntheticVariableValues :: !ValidatedSyntheticVariables
+  , _rlqpQueryVariableTypes      :: !GV.ReusableVariableTypes
   } deriving (Show)
 $(J.deriveToJSON (J.aesonDrop 4 J.snakeCase) ''ReusableLiveQueryPlan)
 
@@ -170,18 +257,20 @@ buildLiveQueryPlan
 buildLiveQueryPlan pgExecCtx fieldAlias astUnresolved varTypes = do
   userInfo <- asks getter
 
-  (astResolved, annVarVals) <- flip runStateT mempty $
-    GR.traverseQueryRootFldAST toMultiplexedQueryVar astUnresolved
+  (astResolved, (queryVariableValues, syntheticVariableValues)) <- flip runStateT mempty $
+    GR.traverseQueryRootFldAST resolveMultiplexedValue astUnresolved
   let pgQuery = mkMultiplexedQuery $ GR.toPGQuery astResolved
       parameterizedPlan = ParameterizedLiveQueryPlan (userRole userInfo) fieldAlias pgQuery
 
   -- We need to ensure that the values provided for variables
   -- are correct according to Postgres. Without this check
   -- an invalid value for a variable for one instance of the
   -- subscription will take down the entire multiplexed query
-  validatedVars <- validateQueryVariables pgExecCtx annVarVals
-  let plan = LiveQueryPlan parameterizedPlan (userVars userInfo) validatedVars
-      reusablePlan = ReusableLiveQueryPlan parameterizedPlan <$> varTypes
+  validatedQueryVars <- validateVariables pgExecCtx queryVariableValues
+  validatedSyntheticVars <- validateVariables pgExecCtx (toList syntheticVariableValues)
+  let cohortVariables = CohortVariables (userVars userInfo) validatedQueryVars validatedSyntheticVars
+      plan = LiveQueryPlan parameterizedPlan cohortVariables
+      reusablePlan = ReusableLiveQueryPlan parameterizedPlan validatedSyntheticVars <$> varTypes
   pure (plan, reusablePlan)
 
 reuseLiveQueryPlan
@@ -192,7 +281,23 @@ reuseLiveQueryPlan
   -> ReusableLiveQueryPlan
   -> m LiveQueryPlan
 reuseLiveQueryPlan pgExecCtx sessionVars queryVars reusablePlan = do
-  let ReusableLiveQueryPlan parameterizedPlan varTypes = reusablePlan
-  annVarVals <- GV.validateVariablesForReuse varTypes queryVars
-  validatedVars <- validateQueryVariables pgExecCtx annVarVals
-  pure $ LiveQueryPlan parameterizedPlan sessionVars validatedVars
+  let ReusableLiveQueryPlan parameterizedPlan syntheticVars queryVarTypes = reusablePlan
+  annVarVals <- GV.validateVariablesForReuse queryVarTypes queryVars
+  validatedVars <- validateVariables pgExecCtx annVarVals
+  pure $ LiveQueryPlan parameterizedPlan (CohortVariables sessionVars validatedVars syntheticVars)
+
+data LiveQueryPlanExplanation
+  = LiveQueryPlanExplanation
+  { _lqpeSql  :: !Text
+  , _lqpePlan :: ![Text]
+  } deriving (Show)
+$(J.deriveToJSON (J.aesonDrop 5 J.snakeCase) ''LiveQueryPlanExplanation)
+
+explainLiveQueryPlan :: (MonadTx m, MonadIO m) => LiveQueryPlan -> m LiveQueryPlanExplanation
+explainLiveQueryPlan plan = do
+  let parameterizedPlan = _lqpParameterizedPlan plan
+      queryText = Q.getQueryText . unMultiplexedQuery $ _plqpQuery parameterizedPlan
+      explainQuery = Q.fromText $ "EXPLAIN (FORMAT TEXT) " <> queryText
+  cohortId <- newCohortId
+  explanationLines <- map runIdentity <$> executeQuery explainQuery [(cohortId, _lqpVariables plan)]
+  pure $ LiveQueryPlanExplanation queryText explanationLines