From 5d0ee6a8a991ef9c887ee5c3f242a7d10d5ceab8 Mon Sep 17 00:00:00 2001 From: Juraj Majerik Date: Mon, 16 Sep 2024 14:46:03 +0200 Subject: [PATCH 1/2] init --- frontend/src/queries/schema.json | 22 ++---- frontend/src/queries/schema.ts | 4 +- latest_migrations.manifest | 2 +- .../experiment_funnel_query_runner.py | 10 ++- .../experiment_trend_query_runner.py | 12 ++- .../test_experiment_funnel_query_runner.py | 63 +++++++++++---- .../test_experiment_trend_query_runner.py | 79 +++++++++++++------ posthog/migrations/0472_experiment_metrics.py | 17 ++++ posthog/models/experiment.py | 2 + posthog/schema.py | 4 +- 10 files changed, 153 insertions(+), 62 deletions(-) create mode 100644 posthog/migrations/0472_experiment_metrics.py diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 93a5a0839e171..610d7d7a84ba0 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -4536,6 +4536,9 @@ "ExperimentFunnelQuery": { "additionalProperties": false, "properties": { + "experiment_id": { + "type": "integer" + }, "kind": { "const": "ExperimentFunnelQuery", "type": "string" @@ -4549,15 +4552,9 @@ }, "source": { "$ref": "#/definitions/FunnelsQuery" - }, - "variants": { - "items": { - "type": "string" - }, - "type": "array" } }, - "required": ["kind", "source", "variants"], + "required": ["experiment_id", "kind", "source"], "type": "object" }, "ExperimentFunnelQueryResponse": { @@ -4583,6 +4580,9 @@ "count_source": { "$ref": "#/definitions/TrendsQuery" }, + "experiment_id": { + "type": "integer" + }, "exposure_source": { "$ref": "#/definitions/TrendsQuery" }, @@ -4596,15 +4596,9 @@ }, "response": { "$ref": "#/definitions/ExperimentTrendQueryResponse" - }, - "variants": { - "items": { - "type": "string" - }, - "type": "array" } }, - "required": ["count_source", "exposure_source", "kind", "variants"], + "required": ["count_source", "experiment_id", "exposure_source", "kind"], "type": "object" }, "ExperimentTrendQueryResponse": { diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 15d132f44ad8f..0b1d9e3c791d8 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -1577,14 +1577,14 @@ export interface ExperimentFunnelQueryResponse { export interface ExperimentFunnelQuery extends DataNode { kind: NodeKind.ExperimentFunnelQuery source: FunnelsQuery - variants: string[] + experiment_id: integer } export interface ExperimentTrendQuery extends DataNode { kind: NodeKind.ExperimentTrendQuery count_source: TrendsQuery exposure_source: TrendsQuery - variants: string[] + experiment_id: integer } /** diff --git a/latest_migrations.manifest b/latest_migrations.manifest index 7a0780defe6bd..d0fb78b374262 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name ee: 0016_rolemembership_organization_member otp_static: 0002_throttling otp_totp: 0002_auto_20190420_0723 -posthog: 0471_webexperiment_experiment_type_experiment_variants +posthog: 0472_experiment_metrics sessions: 0001_initial social_django: 0010_uid_db_index two_factor: 0007_auto_20201201_1019 diff --git a/posthog/hogql_queries/experiment_funnel_query_runner.py b/posthog/hogql_queries/experiment_funnel_query_runner.py index ef9b2d3879566..36bdc42f0baf5 100644 --- a/posthog/hogql_queries/experiment_funnel_query_runner.py +++ b/posthog/hogql_queries/experiment_funnel_query_runner.py @@ -1,5 +1,6 @@ from posthog.hogql import ast from posthog.hogql_queries.query_runner import QueryRunner +from posthog.models.experiment import Experiment from .insights.funnels.funnels_query_runner import FunnelsQueryRunner from posthog.schema import ( ExperimentFunnelQuery, @@ -14,6 +15,9 @@ class ExperimentFunnelQueryRunner(QueryRunner): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.experiment = Experiment.objects.get(id=self.query.experiment_id) + self.feature_flag = self.experiment.feature_flag + self.query_runner = FunnelsQueryRunner( query=self.query.source, team=self.team, timings=self.timings, limit_context=self.limit_context ) @@ -24,9 +28,9 @@ def calculate(self) -> ExperimentFunnelQueryResponse: return ExperimentFunnelQueryResponse(insight="FUNNELS", results=results) def _process_results(self, funnels_results: list[list[dict[str, Any]]]) -> dict[str, ExperimentVariantFunnelResult]: - variants = self.query.variants + variants = self.feature_flag.variants processed_results = { - variant: ExperimentVariantFunnelResult(success_count=0, failure_count=0) for variant in variants + variant["key"]: ExperimentVariantFunnelResult(success_count=0, failure_count=0) for variant in variants } for result in funnels_results: @@ -34,7 +38,7 @@ def _process_results(self, funnels_results: list[list[dict[str, Any]]]) -> dict[ last_step = result[-1] variant = first_step.get("breakdown_value") variant_str = variant[0] if isinstance(variant, list) else str(variant) - if variant_str in variants: + if variant_str in processed_results: total_count = first_step.get("count", 0) success_count = last_step.get("count", 0) if len(result) > 1 else 0 processed_results[variant_str].success_count = success_count diff --git a/posthog/hogql_queries/experiment_trend_query_runner.py b/posthog/hogql_queries/experiment_trend_query_runner.py index 6961a105b421a..b17d0cbd5bbe8 100644 --- a/posthog/hogql_queries/experiment_trend_query_runner.py +++ b/posthog/hogql_queries/experiment_trend_query_runner.py @@ -2,6 +2,7 @@ from posthog.hogql import ast from posthog.hogql_queries.insights.trends.trends_query_runner import TrendsQueryRunner from posthog.hogql_queries.query_runner import QueryRunner +from posthog.models.experiment import Experiment from posthog.schema import ( ExperimentTrendQuery, ExperimentTrendQueryResponse, @@ -16,6 +17,9 @@ class ExperimentTrendQueryRunner(QueryRunner): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.experiment = Experiment.objects.get(id=self.query.experiment_id) + self.feature_flag = self.experiment.feature_flag + self.query_runner = TrendsQueryRunner( query=self.query.count_source, team=self.team, timings=self.timings, limit_context=self.limit_context ) @@ -68,17 +72,17 @@ def run(query_runner: TrendsQueryRunner, is_parallel: bool): def _process_results( self, count_results: list[dict[str, Any]], exposure_results: list[dict[str, Any]] ) -> dict[str, ExperimentVariantTrendResult]: - variants = self.query.variants - processed_results = {variant: ExperimentVariantTrendResult(count=0, exposure=0) for variant in variants} + variants = self.feature_flag.variants + processed_results = {variant["key"]: ExperimentVariantTrendResult(count=0, exposure=0) for variant in variants} for result in count_results: variant = result.get("breakdown_value") - if variant in variants: + if variant in processed_results: processed_results[variant].count += result.get("count", 0) for result in exposure_results: variant = result.get("breakdown_value") - if variant in variants: + if variant in processed_results: processed_results[variant].exposure += result.get("count", 0) return processed_results diff --git a/posthog/hogql_queries/test/test_experiment_funnel_query_runner.py b/posthog/hogql_queries/test/test_experiment_funnel_query_runner.py index f34b4e401432f..da9d14fb511be 100644 --- a/posthog/hogql_queries/test/test_experiment_funnel_query_runner.py +++ b/posthog/hogql_queries/test/test_experiment_funnel_query_runner.py @@ -1,4 +1,6 @@ from posthog.hogql_queries.experiment_funnel_query_runner import ExperimentFunnelQueryRunner +from posthog.models.experiment import Experiment +from posthog.models.feature_flag.feature_flag import FeatureFlag from posthog.schema import ( BreakdownFilter, EventsNode, @@ -12,11 +14,51 @@ class TestExperimentFunnelQueryRunner(ClickhouseTestMixin, APIBaseTest): - def setUp(self): - super().setUp() - def test_query_runner(self): - feature_flag_property = f"$feature/test-experiment" + feature_flag = FeatureFlag.objects.create( + name="Test experiment flag", + key="test-experiment", + team=self.team, + filters={ + "groups": [{"properties": [], "rollout_percentage": None}], + "multivariate": { + "variants": [ + { + "key": "control", + "name": "Control", + "rollout_percentage": 50, + }, + { + "key": "test", + "name": "Test", + "rollout_percentage": 50, + }, + ] + }, + }, + created_by=self.user, + ) + experiment = Experiment.objects.create( + name="test-experiment", + team=self.team, + feature_flag=feature_flag, + ) + + feature_flag_property = f"$feature/{feature_flag.key}" + + funnels_query = FunnelsQuery( + series=[EventsNode(event="$pageview"), EventsNode(event="purchase")], + dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, + breakdownFilter=BreakdownFilter(breakdown=feature_flag_property), + ) + experiment_query = ExperimentFunnelQuery( + experiment_id=experiment.id, + kind="ExperimentFunnelQuery", + source=funnels_query, + ) + + experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}] + experiment.save() with freeze_time("2020-01-10 12:00:00"): for variant, purchase_count in [("control", 6), ("test", 8)]: @@ -40,17 +82,10 @@ def test_query_runner(self): flush_persons_and_events() - funnels_query = FunnelsQuery( - series=[EventsNode(event="$pageview"), EventsNode(event="purchase")], - dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, - breakdownFilter=BreakdownFilter(breakdown=feature_flag_property), + query_runner = ExperimentFunnelQueryRunner( + query=ExperimentFunnelQuery(**experiment.metrics[0]["query"]), team=self.team ) - experiment_query = ExperimentFunnelQuery( - kind="ExperimentFunnelQuery", source=funnels_query, variants=["control", "test"] - ) - - runner = ExperimentFunnelQueryRunner(query=experiment_query, team=self.team) - result = runner.calculate() + result = query_runner.calculate() self.assertEqual(result.insight, "FUNNELS") self.assertEqual(len(result.results), 2) diff --git a/posthog/hogql_queries/test/test_experiment_trend_query_runner.py b/posthog/hogql_queries/test/test_experiment_trend_query_runner.py index fa562e6e245af..846bffda50a09 100644 --- a/posthog/hogql_queries/test/test_experiment_trend_query_runner.py +++ b/posthog/hogql_queries/test/test_experiment_trend_query_runner.py @@ -1,5 +1,7 @@ from django.test import override_settings from posthog.hogql_queries.experiment_trend_query_runner import ExperimentTrendQueryRunner +from posthog.models.experiment import Experiment +from posthog.models.feature_flag.feature_flag import FeatureFlag from posthog.schema import ( BreakdownFilter, EventsNode, @@ -15,7 +17,58 @@ @override_settings(IN_UNIT_TESTING=True) class TestExperimentTrendQueryRunner(ClickhouseTestMixin, APIBaseTest): def test_query_runner(self): - feature_flag_property = f"$feature/test-experiment" + feature_flag = FeatureFlag.objects.create( + name="Test experiment flag", + key="test-experiment", + team=self.team, + filters={ + "groups": [{"properties": [], "rollout_percentage": None}], + "multivariate": { + "variants": [ + { + "key": "control", + "name": "Control", + "rollout_percentage": 50, + }, + { + "key": "test", + "name": "Test", + "rollout_percentage": 50, + }, + ] + }, + }, + created_by=self.user, + ) + experiment = Experiment.objects.create( + name="test-experiment", + team=self.team, + feature_flag=feature_flag, + ) + + feature_flag_property = f"$feature/{feature_flag.key}" + count_query = TrendsQuery( + kind="TrendsQuery", + series=[EventsNode(event="$pageview")], + dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, + breakdownFilter=BreakdownFilter(breakdown=feature_flag_property), + ) + exposure_query = TrendsQuery( + kind="TrendsQuery", + series=[EventsNode(event="$feature_flag_called")], + dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, + breakdownFilter=BreakdownFilter(breakdown=feature_flag_property), + ) + + experiment_query = ExperimentTrendQuery( + experiment_id=experiment.id, + kind="ExperimentTrendQuery", + count_source=count_query, + exposure_source=exposure_query, + ) + + experiment.metrics = [{"type": "primary", "query": experiment_query.model_dump()}] + experiment.save() with freeze_time("2020-01-10 12:00:00"): # Populate experiment events @@ -40,28 +93,10 @@ def test_query_runner(self): flush_persons_and_events() - count_query = TrendsQuery( - kind="TrendsQuery", - series=[EventsNode(event="$pageview")], - dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, - breakdownFilter=BreakdownFilter(breakdown=feature_flag_property), - ) - exposure_query = TrendsQuery( - kind="TrendsQuery", - series=[EventsNode(event="$feature_flag_called")], - dateRange={"date_from": "2020-01-01", "date_to": "2020-01-14"}, - breakdownFilter=BreakdownFilter(breakdown=feature_flag_property), - ) - - experiment_query = ExperimentTrendQuery( - kind="ExperimentTrendQuery", - count_source=count_query, - exposure_source=exposure_query, - variants=["control", "test"], + query_runner = ExperimentTrendQueryRunner( + query=ExperimentTrendQuery(**experiment.metrics[0]["query"]), team=self.team ) - - runner = ExperimentTrendQueryRunner(query=experiment_query, team=self.team) - result = runner.calculate() + result = query_runner.calculate() self.assertEqual(result.insight, "TRENDS") self.assertEqual(len(result.results), 2) diff --git a/posthog/migrations/0472_experiment_metrics.py b/posthog/migrations/0472_experiment_metrics.py new file mode 100644 index 0000000000000..54fd31c20b4e6 --- /dev/null +++ b/posthog/migrations/0472_experiment_metrics.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.15 on 2024-09-16 12:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("posthog", "0471_webexperiment_experiment_type_experiment_variants"), + ] + + operations = [ + migrations.AddField( + model_name="experiment", + name="metrics", + field=models.JSONField(blank=True, default=list, null=True), + ), + ] diff --git a/posthog/models/experiment.py b/posthog/models/experiment.py index 5c03c36c85eb0..f594c0faf5ed8 100644 --- a/posthog/models/experiment.py +++ b/posthog/models/experiment.py @@ -38,6 +38,8 @@ class ExperimentType(models.TextChoices): type = models.CharField(max_length=40, choices=ExperimentType.choices, null=True, blank=True, default="product") variants = models.JSONField(default=dict, null=True, blank=True) + metrics = models.JSONField(default=list, null=True, blank=True) + def get_feature_flag_key(self): return self.feature_flag.key diff --git a/posthog/schema.py b/posthog/schema.py index 4af2a7339913b..52c5f2f640f9e 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -5086,13 +5086,13 @@ class ExperimentTrendQuery(BaseModel): extra="forbid", ) count_source: TrendsQuery + experiment_id: int exposure_source: TrendsQuery kind: Literal["ExperimentTrendQuery"] = "ExperimentTrendQuery" modifiers: Optional[HogQLQueryModifiers] = Field( default=None, description="Modifiers used when performing the query" ) response: Optional[ExperimentTrendQueryResponse] = None - variants: list[str] class FunnelsQuery(BaseModel): @@ -5502,13 +5502,13 @@ class ExperimentFunnelQuery(BaseModel): model_config = ConfigDict( extra="forbid", ) + experiment_id: int kind: Literal["ExperimentFunnelQuery"] = "ExperimentFunnelQuery" modifiers: Optional[HogQLQueryModifiers] = Field( default=None, description="Modifiers used when performing the query" ) response: Optional[ExperimentFunnelQueryResponse] = None source: FunnelsQuery - variants: list[str] class FunnelPathsFilter(BaseModel): From a0e1c401198a5e2f847d3b3f5155f6d5a079d390 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 13:04:10 +0000 Subject: [PATCH 2/2] Update query snapshots --- posthog/api/test/__snapshots__/test_feature_flag.ambr | 3 ++- .../api/test/__snapshots__/test_organization_feature_flag.ambr | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/posthog/api/test/__snapshots__/test_feature_flag.ambr b/posthog/api/test/__snapshots__/test_feature_flag.ambr index a9f930a55e9b0..57d56d8233aba 100644 --- a/posthog/api/test/__snapshots__/test_feature_flag.ambr +++ b/posthog/api/test/__snapshots__/test_feature_flag.ambr @@ -1785,7 +1785,8 @@ "posthog_experiment"."updated_at", "posthog_experiment"."archived", "posthog_experiment"."type", - "posthog_experiment"."variants" + "posthog_experiment"."variants", + "posthog_experiment"."metrics" FROM "posthog_experiment" WHERE "posthog_experiment"."exposure_cohort_id" = 2 ''' diff --git a/posthog/api/test/__snapshots__/test_organization_feature_flag.ambr b/posthog/api/test/__snapshots__/test_organization_feature_flag.ambr index 826406a32eb6f..91cb4c5b70902 100644 --- a/posthog/api/test/__snapshots__/test_organization_feature_flag.ambr +++ b/posthog/api/test/__snapshots__/test_organization_feature_flag.ambr @@ -1035,7 +1035,8 @@ "posthog_experiment"."updated_at", "posthog_experiment"."archived", "posthog_experiment"."type", - "posthog_experiment"."variants" + "posthog_experiment"."variants", + "posthog_experiment"."metrics" FROM "posthog_experiment" WHERE "posthog_experiment"."feature_flag_id" = 2 '''