feat(hogql): Add paths query runner (#19761)

* Add paths query runner
* Enhance frontend
* Align switch with filter test users

frontend/src/lib/constants.tsx

@@ -165,6 +165,7 @@ export const FEATURE_FLAGS = {
     APPS_AND_EXPORTS_UI: 'apps-and-exports-ui', // owner: @benjackwhite
     SESSION_REPLAY_CORS_PROXY: 'session-replay-cors-proxy', // owner: #team-replay
     HOGQL_INSIGHTS_LIFECYCLE: 'hogql-insights-lifecycle', // owner: @mariusandra
+    HOGQL_INSIGHTS_PATHS: 'hogql-insights-paths', // owner: @webjunkie
     HOGQL_INSIGHTS_RETENTION: 'hogql-insights-retention', // owner: @webjunkie
     HOGQL_INSIGHTS_TRENDS: 'hogql-insights-trends', // owner: @Gilbert09
     HOGQL_INSIGHTS_STICKINESS: 'hogql-insights-stickiness', // owner: @Gilbert09

frontend/src/queries/nodes/DataNode/dataNodeLogic.ts

@@ -324,6 +324,10 @@ export const dataNodeLogic = kea<dataNodeLogicType>([
             (s) => [s.featureFlags],
             (featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_LIFECYCLE],
         ],
+        hogQLInsightsPathsFlagEnabled: [
+            (s) => [s.featureFlags],
+            (featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_PATHS],
+        ],
         hogQLInsightsRetentionFlagEnabled: [
             (s) => [s.featureFlags],
             (featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_RETENTION],

frontend/src/queries/query.ts

@@ -29,6 +29,7 @@ import {
     isInsightQueryNode,
     isInsightVizNode,
     isLifecycleQuery,
+    isPathsQuery,
     isPersonsNode,
     isRetentionQuery,
     isStickinessQuery,
@@ -150,6 +151,9 @@ export async function query<N extends DataNode = DataNode>(
     const hogQLInsightsLifecycleFlagEnabled = Boolean(
         featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.HOGQL_INSIGHTS_LIFECYCLE]
     )
+    const hogQLInsightsPathsFlagEnabled = Boolean(
+        featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.HOGQL_INSIGHTS_PATHS]
+    )
     const hogQLInsightsRetentionFlagEnabled = Boolean(
         featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.HOGQL_INSIGHTS_RETENTION]
     )
@@ -201,6 +205,7 @@ export async function query<N extends DataNode = DataNode>(
         } else if (isInsightQueryNode(queryNode)) {
             if (
                 (hogQLInsightsLifecycleFlagEnabled && isLifecycleQuery(queryNode)) ||
+                (hogQLInsightsPathsFlagEnabled && isPathsQuery(queryNode)) ||
                 (hogQLInsightsRetentionFlagEnabled && isRetentionQuery(queryNode)) ||
                 (hogQLInsightsTrendsFlagEnabled && isTrendsQuery(queryNode)) ||
                 (hogQLInsightsStickinessFlagEnabled && isStickinessQuery(queryNode))

frontend/src/queries/schema.json

@@ -2195,7 +2195,7 @@
             "additionalProperties": false,
             "properties": {
                 "edgeLimit": {
-                    "type": "number"
+                    "type": "integer"
                 },
                 "endPoint": {
                     "type": "string"
@@ -2225,10 +2225,10 @@
                     "type": "array"
                 },
                 "maxEdgeWeight": {
-                    "type": "number"
+                    "type": "integer"
                 },
                 "minEdgeWeight": {
-                    "type": "number"
+                    "type": "integer"
                 },
                 "pathGroupings": {
                     "items": {
@@ -2246,7 +2246,7 @@
                     "type": "string"
                 },
                 "stepLimit": {
-                    "type": "number"
+                    "type": "integer"
                 }
             },
             "type": "object"
@@ -2256,7 +2256,7 @@
             "description": "`PathsFilterType` minus everything inherited from `FilterType` and persons modal related params",
             "properties": {
                 "edge_limit": {
-                    "type": "number"
+                    "type": "integer"
                 },
                 "end_point": {
                     "type": "string"
@@ -2286,10 +2286,10 @@
                     "type": "array"
                 },
                 "max_edge_weight": {
-                    "type": "number"
+                    "type": "integer"
                 },
                 "min_edge_weight": {
-                    "type": "number"
+                    "type": "integer"
                 },
                 "path_groupings": {
                     "items": {
@@ -2310,7 +2310,7 @@
                     "type": "string"
                 },
                 "step_limit": {
-                    "type": "number"
+                    "type": "integer"
                 }
             },
             "type": "object"
@@ -2352,12 +2352,46 @@
                     ],
                     "description": "Property filters for all series"
                 },
+                "response": {
+                    "$ref": "#/definitions/PathsQueryResponse"
+                },
                 "samplingFactor": {
                     "description": "Sampling rate",
                     "type": ["number", "null"]
                 }
             },
-            "required": ["kind"],
+            "required": ["kind", "pathsFilter"],
+            "type": "object"
+        },
+        "PathsQueryResponse": {
+            "additionalProperties": false,
+            "properties": {
+                "hogql": {
+                    "type": "string"
+                },
+                "is_cached": {
+                    "type": "boolean"
+                },
+                "last_refresh": {
+                    "type": "string"
+                },
+                "next_allowed_client_refresh": {
+                    "type": "string"
+                },
+                "results": {
+                    "items": {
+                        "type": "object"
+                    },
+                    "type": "array"
+                },
+                "timings": {
+                    "items": {
+                        "$ref": "#/definitions/QueryTiming"
+                    },
+                    "type": "array"
+                }
+            },
+            "required": ["results"],
             "type": "object"
         },
         "PersonPropertyFilter": {
@@ -3019,6 +3053,37 @@
                     "required": ["results"],
                     "type": "object"
                 },
+                {
+                    "additionalProperties": false,
+                    "properties": {
+                        "hogql": {
+                            "type": "string"
+                        },
+                        "is_cached": {
+                            "type": "boolean"
+                        },
+                        "last_refresh": {
+                            "type": "string"
+                        },
+                        "next_allowed_client_refresh": {
+                            "type": "string"
+                        },
+                        "results": {
+                            "items": {
+                                "type": "object"
+                            },
+                            "type": "array"
+                        },
+                        "timings": {
+                            "items": {
+                                "$ref": "#/definitions/QueryTiming"
+                            },
+                            "type": "array"
+                        }
+                    },
+                    "required": ["results"],
+                    "type": "object"
+                },
                 {
                     "additionalProperties": {
                         "items": {

frontend/src/queries/schema.ts

@@ -607,6 +607,9 @@ export interface RetentionQuery extends InsightsQueryBase {
     retentionFilter: RetentionFilter
 }
 
+export interface PathsQueryResponse extends QueryResponse {
+    results: Record<string, any>[]
+}
 /** `PathsFilterType` minus everything inherited from `FilterType` and persons modal related params */
 export type PathsFilterLegacy = Omit<
     PathsFilterType,
@@ -632,8 +635,9 @@ export type PathsFilter = {
 
 export interface PathsQuery extends InsightsQueryBase {
     kind: NodeKind.PathsQuery
+    response?: PathsQueryResponse
     /** Properties specific to the paths insight */
-    pathsFilter?: PathsFilter
+    pathsFilter: PathsFilter
 }
 
 /** `StickinessFilterType` minus everything inherited from `FilterType` and persons modal related params

frontend/src/scenes/insights/EditorFilters/PathsAdvanced.tsx

@@ -1,12 +1,9 @@
 import { LemonDivider, LemonInput } from '@posthog/lemon-ui'
 import { useActions, useValues } from 'kea'
 import { PayGateMini } from 'lib/components/PayGateMini/PayGateMini'
-import { IconSettings } from 'lib/lemon-ui/icons'
 import { LemonLabel } from 'lib/lemon-ui/LemonLabel/LemonLabel'
-import { Link } from 'lib/lemon-ui/Link'
 import { useState } from 'react'
 import { pathsDataLogic } from 'scenes/paths/pathsDataLogic'
-import { urls } from 'scenes/urls'
 
 import { AvailableFeature, EditorFilterProps, PathEdgeParameters } from '~/types'
 
@@ -61,7 +58,7 @@ export function PathsAdvanced({ insightProps, ...rest }: EditorFilterProps): JSX
                 >
                     Number of people on each path
                 </LemonLabel>
-                <div>
+                <div className="flex items-baseline">
                     <span className="mr-2">Between</span>
                     <LemonInput
                         type="number"
@@ -95,7 +92,6 @@ export function PathsAdvanced({ insightProps, ...rest }: EditorFilterProps): JSX
                 <div>
                     <div className="flex items-center my-2">
                         <LemonLabel
-                            showOptional
                             info={
                                 <>
                                     Cleaning rules are an advanced feature that uses regex to normalize URLS for paths
@@ -106,13 +102,6 @@ export function PathsAdvanced({ insightProps, ...rest }: EditorFilterProps): JSX
                         >
                             Path Cleaning Rules
                         </LemonLabel>
-                        <Link
-                            className="flex items-center ml-2"
-                            to={urls.settings('project-product-analytics', 'path-cleaning')}
-                        >
-                            <IconSettings fontSize="16" className="mr-0.5" />
-                            Configure Project Rules
-                        </Link>
                     </div>
                     <PathCleaningFilter insightProps={insightProps} {...rest} />
                 </div>

frontend/src/scenes/insights/filters/PathCleaningFilter.tsx

@@ -1,9 +1,11 @@
-import { LemonSwitch } from '@posthog/lemon-ui'
+import { LemonButton, LemonSwitch } from '@posthog/lemon-ui'
 import { useActions, useValues } from 'kea'
 import { PathCleanFilters } from 'lib/components/PathCleanFilters/PathCleanFilters'
+import { IconSettings } from 'lib/lemon-ui/icons'
 import { Tooltip } from 'lib/lemon-ui/Tooltip'
 import { pathsDataLogic } from 'scenes/paths/pathsDataLogic'
 import { teamLogic } from 'scenes/teamLogic'
+import { urls } from 'scenes/urls'
 
 import { EditorFilterProps } from '~/types'
 
@@ -26,18 +28,30 @@ export function PathCleaningFilter({ insightProps }: EditorFilterProps): JSX.Ele
                 title={
                     hasFilters
                         ? 'Clean paths based using regex replacement.'
-                        : "You don't have path cleaning filters. Click the gear icon to configure it."
+                        : "You don't have path cleaning filters. Configure via the gear icon."
                 }
             >
                 {/* This div is necessary for the tooltip to work. */}
-                <div className="inline-block mt-4">
+                <div className="inline-block mt-4 w-full">
                     <LemonSwitch
                         disabled={!hasFilters}
                         checked={hasFilters ? pathReplacements || false : false}
                         onChange={(checked: boolean) => {
                             updateInsightFilter({ pathReplacements: checked })
                         }}
-                        label="Apply global path URL cleaning"
+                        label={
+                            <div className="flex items-center">
+                                <span>Apply global path URL cleaning</span>
+                                <LemonButton
+                                    icon={<IconSettings />}
+                                    to={urls.settings('project-product-analytics', 'path-cleaning')}
+                                    size="small"
+                                    noPadding
+                                    className="ml-1"
+                                />
+                            </div>
+                        }
+                        fullWidth
                         bordered
                     />
                 </div>

frontend/src/scenes/insights/insightVizDataLogic.ts

@@ -423,6 +423,7 @@ const handleQuerySourceUpdateSideEffects = (
      * Date range change side effects.
      */
     if (
+        !isPathsQuery(currentState) && // TODO: Apply side logic more elegantly
         update.dateRange &&
         update.dateRange.date_from &&
         (update.dateRange.date_from !== currentState.dateRange?.date_from ||

frontend/src/types.ts

@@ -1872,11 +1872,15 @@ export interface PathsFilterType extends FilterType {
     funnel_paths?: FunnelPathType
     funnel_filter?: Record<string, any> // Funnel Filter used in Paths
     exclude_events?: string[] // Paths Exclusion type
+    /** @asType integer */
     step_limit?: number // Paths Step Limit
     path_replacements?: boolean
     local_path_cleaning_filters?: PathCleaningFilter[]
+    /** @asType integer */
     edge_limit?: number | undefined // Paths edge limit
+    /** @asType integer */
     min_edge_weight?: number | undefined // Paths
+    /** @asType integer */
     max_edge_weight?: number | undefined // Paths
 
     // persons only

mypy-baseline.txt

@@ -365,7 +365,6 @@ posthog/hogql/query.py:0: error: Incompatible types in assignment (expression ha
 posthog/hogql/query.py:0: error: Argument 1 to "get_default_limit_for_context" has incompatible type "LimitContext | None"; expected "LimitContext"  [arg-type]
 posthog/hogql/query.py:0: error: "SelectQuery" has no attribute "select_queries"  [attr-defined]
 posthog/hogql/query.py:0: error: Subclass of "SelectQuery" and "SelectUnionQuery" cannot exist: would have incompatible method signatures  [unreachable]
-posthog/hogql_queries/query_runner.py:0: error: Incompatible types in assignment (expression has type "HogQLQuery | TrendsQuery | LifecycleQuery | InsightActorsQuery | EventsQuery | ActorsQuery | RetentionQuery | SessionsTimelineQuery | WebOverviewQuery | WebTopClicksQuery | WebStatsTableQuery | StickinessQuery | BaseModel | dict[str, Any]", variable has type "HogQLQuery | TrendsQuery | LifecycleQuery | InsightActorsQuery | EventsQuery | ActorsQuery | RetentionQuery | SessionsTimelineQuery | WebOverviewQuery | WebTopClicksQuery | WebStatsTableQuery | StickinessQuery")  [assignment]
 posthog/hogql_queries/insights/trends/breakdown_values.py:0: error: Item "SelectUnionQuery" of "SelectQuery | SelectUnionQuery" has no attribute "select"  [union-attr]
 posthog/hogql_queries/insights/trends/breakdown_values.py:0: error: Value of type "list[Any] | None" is not indexable  [index]
 posthog/hogql_queries/sessions_timeline_query_runner.py:0: error: Statement is unreachable  [unreachable]
@@ -510,7 +509,6 @@ posthog/session_recordings/queries/session_recording_list_from_replay_summary.py
 posthog/session_recordings/queries/session_recording_list_from_replay_summary.py:0: note: If the method is meant to be abstract, use @abc.abstractmethod
 posthog/session_recordings/queries/session_recording_list_from_replay_summary.py:0: error: Incompatible types in assignment (expression has type "PersonOnEventsMode", variable has type "PersonsOnEventsMode | None")  [assignment]
 posthog/session_recordings/queries/session_recording_list_from_replay_summary.py:0: error: Incompatible types in assignment (expression has type "PersonOnEventsMode", variable has type "PersonsOnEventsMode | None")  [assignment]
-posthog/hogql_queries/test/test_query_runner.py:0: error: Incompatible default for argument "query_class" (default has type "type[TestQuery]", argument has type "type[HogQLQuery] | type[TrendsQuery] | type[LifecycleQuery] | type[InsightActorsQuery] | type[EventsQuery] | type[ActorsQuery] | type[RetentionQuery] | type[SessionsTimelineQuery] | type[WebOverviewQuery] | type[WebTopClicksQuery] | type[WebStatsTableQuery] | type[StickinessQuery]")  [assignment]
 posthog/hogql_queries/test/test_query_runner.py:0: error: Variable "TestQueryRunner" is not valid as a type  [valid-type]
 posthog/hogql_queries/test/test_query_runner.py:0: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
 posthog/hogql_queries/test/test_query_runner.py:0: error: Invalid base class "TestQueryRunner"  [misc]

posthog/api/services/query.py

@@ -30,12 +30,14 @@
     TimeToSeeDataSessionsQuery,
     TimeToSeeDataQuery,
     StickinessQuery,
+    PathsQuery,
 )
 
 logger = structlog.get_logger(__name__)
 
 QUERY_WITH_RUNNER = (
     LifecycleQuery
+    | PathsQuery
     | RetentionQuery
     | StickinessQuery
     | TrendsQuery

posthog/api/test/__snapshots__/test_query.ambr

@@ -427,7 +427,7 @@
   FROM events
   WHERE and(equals(events.team_id, 2), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC')))
   GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'key'), ''), 'null'), '^"|"$', '')
-  HAVING and(ifNull(greater(count(), 1), 0))
+  HAVING ifNull(greater(count(), 1), 0)
   ORDER BY count() DESC
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
@@ -458,7 +458,7 @@
   FROM events
   WHERE and(equals(events.team_id, 2), less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC')))
   GROUP BY nullIf(nullIf(events.mat_key, ''), 'null')
-  HAVING and(ifNull(greater(count(), 1), 0))
+  HAVING ifNull(greater(count(), 1), 0)
   ORDER BY count() DESC
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,

posthog/hogql/functions/mapping.py

@@ -191,6 +191,7 @@ class HogQLFunctionMeta:
     "toMinute": HogQLFunctionMeta("toMinute", 1, 1),
     "toSecond": HogQLFunctionMeta("toSecond", 1, 1),
     "toUnixTimestamp": HogQLFunctionMeta("toUnixTimestamp", 1, 2),
+    "toUnixTimestamp64Milli": HogQLFunctionMeta("toUnixTimestamp64Milli", 1, 1),
     "toStartOfYear": HogQLFunctionMeta("toStartOfYear", 1, 1),
     "toStartOfISOYear": HogQLFunctionMeta("toStartOfISOYear", 1, 1),
     "toStartOfQuarter": HogQLFunctionMeta("toStartOfQuarter", 1, 1),

posthog/hogql/printer.py

@@ -467,9 +467,13 @@ def visit_arithmetic_operation(self, node: ast.ArithmeticOperation):
             raise HogQLException(f"Unknown ArithmeticOperationOp {node.op}")
 
     def visit_and(self, node: ast.And):
+        if len(node.exprs) == 1:
+            return self.visit(node.exprs[0])
         return f"and({', '.join([self.visit(expr) for expr in node.exprs])})"
 
     def visit_or(self, node: ast.Or):
+        if len(node.exprs) == 1:
+            return self.visit(node.exprs[0])
         return f"or({', '.join([self.visit(expr) for expr in node.exprs])})"
 
     def visit_not(self, node: ast.Not):