[dashboard] UI experiments

components/dashboard/src/Analytics.tsx

@@ -8,44 +8,70 @@ import { getGitpodService } from "./service/service";
 import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
 import Cookies from "js-cookie";
 import { v4 } from "uuid";
+import { Experiment } from "./experiments";
 
 
 export type Event = "invite_url_requested" | "organisation_authorised";
+type InternalEvent = Event | "path_changed" | "dashboard_clicked";
+
+export type EventProperties =
+    TrackOrgAuthorised
+  | TrackInviteUrlRequested
+;
+type InternalEventProperties = TrackUIExperiments & (
+    EventProperties
+  | TrackDashboardClick
+  | TrackPathChanged
+);
+
+export interface TrackOrgAuthorised {
+  installation_id: string,
+  setup_action: string | undefined,
+}
+
+export interface TrackInviteUrlRequested {
+  invite_url: string,
+}
 
-export type TrackingMsg = {
+interface TrackDashboardClick {
   dnt?: boolean,
   path: string,
   button_type?: string,
   label?: string,
-  destination?: string
+  destination?: string,
+};
+
+interface TrackPathChanged {
+  prev: string,
+  path: string,
 }
 
-//call this to track all events outside of button and anchor clicks
-export const trackEvent = (event: Event, properties: any) => {
-  getGitpodService().server.trackEvent({
-    event: event,
-    properties: properties
-  })
+interface TrackUIExperiments {
+  ui_experiments?: string[],
 }
 
-export const getAnonymousId = (): string => {
-  let anonymousId = Cookies.get('ajs_anonymous_id');
-  if (anonymousId) {
-    return anonymousId.replace(/^"(.+(?="$))"$/, '$1'); //strip enclosing double quotes before returning
-  }
-  else {
-    anonymousId = v4();
-    Cookies.set('ajs_anonymous_id', anonymousId, {domain: '.'+window.location.hostname, expires: 365});
-  };
-  return anonymousId;
+//call this to track all events outside of button and anchor clicks
+export const trackEvent = (event: Event, properties: EventProperties) => {
+  trackEventInternal(event, properties);
 }
 
+const trackEventInternal = (event: InternalEvent, properties: InternalEventProperties, userKnown?: boolean) => {
+  properties.ui_experiments = Experiment.getAsArray();
+
+  getGitpodService().server.trackEvent({
+    //if the user is authenticated, let server determine the id. else, pass anonymousId explicitly.
+    anonymousId: userKnown ? undefined : getAnonymousId(),
+    event,
+    properties,
+  });
+};
+
 export const trackButtonOrAnchor = (target: HTMLAnchorElement | HTMLButtonElement | HTMLDivElement, userKnown: boolean) => {
   //read manually passed analytics props from 'data-analytics' attribute of event target
-  let passedProps: TrackingMsg | undefined;
+  let passedProps: TrackDashboardClick | undefined;
   if (target.dataset.analytics) {
     try {
-      passedProps = JSON.parse(target.dataset.analytics) as TrackingMsg;
+      passedProps = JSON.parse(target.dataset.analytics) as TrackDashboardClick;
       if (passedProps.dnt) {
         return;
       }
@@ -55,10 +81,10 @@ export const trackButtonOrAnchor = (target: HTMLAnchorElement | HTMLButtonElemen
 
   }
 
-  let trackingMsg: TrackingMsg = {
+  let trackingMsg: TrackDashboardClick = {
     path: window.location.pathname,
     label: target.textContent || undefined
-  }
+  };
 
   if (target instanceof HTMLButtonElement || target instanceof HTMLDivElement) {
     //parse button data
@@ -79,60 +105,60 @@ export const trackButtonOrAnchor = (target: HTMLAnchorElement | HTMLButtonElemen
     trackingMsg.destination = anchor.href;
   }
 
-  const getAncestorProps = (curr: HTMLElement | null): TrackingMsg | undefined => {
+  const getAncestorProps = (curr: HTMLElement | null): TrackDashboardClick | undefined => {
     if (!curr || curr instanceof Document) {
       return;
     }
-    const ancestorProps: TrackingMsg | undefined = getAncestorProps(curr.parentElement);
-    const currProps = JSON.parse(curr.dataset.analytics || "{}");
-    return {...ancestorProps, ...currProps} as TrackingMsg;
+    const ancestorProps: TrackDashboardClick | undefined = getAncestorProps(curr.parentElement);
+    const currProps = JSON.parse(curr.dataset.analytics || "{}") as TrackDashboardClick;
+    return {...ancestorProps, ...currProps};
   }
 
   const ancestorProps = getAncestorProps(target);
 
   //props that were passed directly to the event target take precedence over those passed to ancestor elements, which take precedence over those implicitly determined.
   trackingMsg = {...trackingMsg, ...ancestorProps, ...passedProps};
 
-  //if the user is authenticated, let server determine the id. else, pass anonymousId explicitly.
-  if (userKnown) {
-    getGitpodService().server.trackEvent({
-      event: "dashboard_clicked",
-      properties: trackingMsg
-    });
-  } else {
-    getGitpodService().server.trackEvent({
-      anonymousId: getAnonymousId(),
-      event: "dashboard_clicked",
-      properties: trackingMsg
-    });
-  }
+  trackEventInternal("dashboard_clicked", trackingMsg, userKnown);
 }
 
 //call this when the path changes. Complete page call is unnecessary for SPA after initial call
-export const trackPathChange = (props: { prev: string, path: string }) => {
-  getGitpodService().server.trackEvent({
-    event: "path_changed",
-    properties: props
-  });
+export const trackPathChange = (props: TrackPathChanged) => {
+  trackEventInternal("path_changed", props);
 }
 
+
+type TrackLocationProperties = TrackUIExperiments & {
+  referrer: string,
+  path: string,
+  host: string,
+  url: string,
+};
+
 export const trackLocation = async (userKnown: boolean) => {
-  const props = {
+  const props: TrackLocationProperties = {
     referrer: document.referrer,
     path: window.location.pathname,
     host: window.location.hostname,
-    url: window.location.href
-  }
-  if (userKnown) {
-    //if the user is known, make server call
-    getGitpodService().server.trackLocation({
-      properties: props
-    });
-  } else {
-    //make privacy preserving page call (automatically interpreted as such by server if anonymousId is passed)
-    getGitpodService().server.trackLocation({
-      anonymousId: getAnonymousId(),
-      properties: props
-    });
+    url: window.location.href,
+    ui_experiments: Experiment.getAsArray(),
+  };
+
+  getGitpodService().server.trackLocation({
+    //if the user is authenticated, let server determine the id. else, pass anonymousId explicitly.
+    anonymousId: userKnown ? undefined : getAnonymousId(),
+    properties: props
+  });
+}
+
+const getAnonymousId = (): string => {
+  let anonymousId = Cookies.get('ajs_anonymous_id');
+  if (anonymousId) {
+    return anonymousId.replace(/^"(.+(?="$))"$/, '$1'); //strip enclosing double quotes before returning
   }
+  else {
+    anonymousId = v4();
+    Cookies.set('ajs_anonymous_id', anonymousId, {domain: '.'+window.location.hostname, expires: 365});
+  };
+  return anonymousId;
 }

components/dashboard/src/App.tsx

@@ -20,6 +20,7 @@ import { useHistory } from 'react-router-dom';
 import { trackButtonOrAnchor, trackPathChange, trackLocation } from './Analytics';
 import { User } from '@gitpod/gitpod-protocol';
 import * as GitpodCookie from '@gitpod/gitpod-protocol/lib/util/gitpod-cookie';
+import { Experiment } from './experiments';
 
 const Setup = React.lazy(() => import(/* webpackPrefetch: true */ './Setup'));
 const Workspaces = React.lazy(() => import(/* webpackPrefetch: true */ './workspaces/Workspaces'));
@@ -167,6 +168,11 @@ function App() {
     }, []);
 
     // listen and notify Segment of client-side path updates
+    useEffect(() => {
+        // Choose which experiments to run for this session/user
+        Experiment.set(Experiment.seed(true));
+    })
+
     useEffect(() => {
         return history.listen((location: any) => {
             const path = window.location.pathname;

components/dashboard/src/Login.tsx

@@ -59,6 +59,7 @@ export function Login() {
 
     const authorizeSuccessful = async (payload?: string) => {
         updateUser().catch(console.error);
+
         // Check for a valid returnTo in payload
         const safeReturnTo = getSafeURLRedirect(payload);
         if (safeReturnTo) {

components/dashboard/src/experiments.ts

@@ -0,0 +1,85 @@
+/**
+ * Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+const UI_EXPERIMENTS_KEY = "gitpod-ui-experiments";
+
+/**
+ * This enables UI-experiments: Dashboard-local changes that we'd like to try out and get some feedback on/validate
+ * our assumptions on via mixpanel before we roll them out to everyone. The motivation is to make UI development more
+ * data-driven then the current approach.
+ *
+ * An experiment is supposed to:
+ *  - only be applied to a subset of users, so we are guaranteed to always have a control group (configurable per experiment)
+ *  - is dashboard/component local: it's not a mechnanism to cross the API boundary, mostly even component-local
+ *  - is stable per session/client
+ *  - is defined in code (see below): adding/deprecating experiments requires a deployment
+ * It is NOT supposed to:
+ *  - big a way to roll-out big, completly new features
+ *  - have a lot of different experiments in parallel (too noisy)
+ *
+ * Questions:
+ *  - multiple experiments per user/time
+ */
+const Experiments = {
+    /**
+     * Experiment "example" will be activate on login for 10% of all clients.
+     */
+    // "example": 0.1,
+};
+const ExperimentsSet = new Set(Object.keys(Experiments)) as Set<Experiment>;
+export type Experiment = keyof (typeof Experiments);
+
+export namespace Experiment {
+    export function seed(keepCurrent: boolean): Set<Experiment> {
+        const current = keepCurrent ? get() : undefined;
+
+        // add all current experiments to ensure stability
+        const result = new Set<Experiment>([...(current || [])].filter(e => ExperimentsSet.has(e)));
+
+        // identify all new experiments and add if random
+        const newExperiment = new Set<Experiment>([...ExperimentsSet].filter(e => !result.has(e)));
+        for (const e of newExperiment) {
+            if (Math.random() < Experiments[e]) {
+                result.add(e);
+            }
+        }
+
+        return result;
+    }
+
+    export function set(set: Set<Experiment>): void {
+        try {
+            const arr = Array.from(set);
+            window.localStorage.setItem(UI_EXPERIMENTS_KEY, JSON.stringify(arr));
+        } catch (err) {
+            console.error(`error setting ${UI_EXPERIMENTS_KEY}`, err);
+        }
+    }
+
+    export function has(experiment: Experiment): boolean {
+        const set = get();
+        if (!set) {
+            return false;
+        }
+        return set.has(experiment);
+    }
+
+    export function get(): Set<Experiment> | undefined {
+        const arr = window.localStorage.getItem(UI_EXPERIMENTS_KEY);
+        if (arr === null) {
+            return undefined;
+        }
+        return new Set(arr) as Set<Experiment>;
+    }
+
+    export function getAsArray(): Experiment[] {
+        const set = get();
+        if (!set) {
+            return [];
+        }
+        return Array.from(set);
+    }
+}