fix: login/utm param bug fixes and test coverage (#23787)

packages/app/cypress/e2e/runs.cy.ts

@@ -75,9 +75,20 @@ describe('App: Runs', { viewportWidth: 1200 }, () => {
     it('clicking the login button will open the login modal', () => {
       cy.visitApp()
       moveToRunsPage()
-      cy.contains('Log In').click()
+      cy.contains(defaultMessages.runs.connect.buttonUser).click()
+      cy.withCtx((ctx, o) => {
+        o.sinon.spy(ctx._apis.authApi, 'logIn')
+      })
+
       cy.findByRole('dialog', { name: 'Log in to Cypress' }).within(() => {
-        cy.get('button').contains('Log In')
+        cy.contains('button', 'Log In').click()
+      })
+
+      cy.withCtx((ctx, o) => {
+        // validate utmSource
+        expect((ctx._apis.authApi.logIn as SinonStub).lastCall.args[1]).to.eq('Binary: App')
+        // validate utmMedium
+        expect((ctx._apis.authApi.logIn as SinonStub).lastCall.args[2]).to.eq('Runs Tab')
       })
     })
 
@@ -239,6 +250,7 @@ describe('App: Runs', { viewportWidth: 1200 }, () => {
       moveToRunsPage()
 
       cy.withCtx(async (ctx, options) => {
+        ctx.coreData.app.browserStatus = 'open'
         options.sinon.stub(ctx._apis.electronApi, 'isMainWindowFocused').returns(false)
         options.sinon.stub(ctx._apis.authApi, 'logIn').callsFake(async (onMessage) => {
           setTimeout(() => {
@@ -272,8 +284,8 @@ describe('App: Runs', { viewportWidth: 1200 }, () => {
   })
 
   context('Runs - Create Project', () => {
-    it('when a project is created, injects new projectId into the config file', () => {
-      cy.remoteGraphQLIntercept(async (obj) => {
+    it('when a project is created, injects new projectId into the config file, and sends expected UTM params', () => {
+      cy.remoteGraphQLIntercept((obj) => {
         if (obj.operationName === 'SelectCloudProjectModal_CreateCloudProject_cloudProjectCreate') {
           obj.result.data!.cloudProjectCreate = {
             slug: 'newProjectId',
@@ -290,7 +302,9 @@ describe('App: Runs', { viewportWidth: 1200 }, () => {
       cy.loginUser()
       cy.visitApp()
 
-      cy.withCtx(async (ctx) => {
+      cy.withCtx(async (ctx, o) => {
+        o.sinon.spy(ctx.cloud, 'executeRemoteGraphQL')
+
         const config = await ctx.project.getConfig()
 
         expect(config.projectId).to.not.equal('newProjectId')
@@ -305,6 +319,12 @@ describe('App: Runs', { viewportWidth: 1200 }, () => {
         const config = await ctx.project.getConfig()
 
         expect(config.projectId).to.equal('newProjectId')
+        expect(ctx.cloud.executeRemoteGraphQL).to.have.been.calledWithMatch({
+          fieldName: 'cloudProjectCreate',
+          operationVariables: {
+            medium: 'Runs Tab',
+            source: 'Binary: App',
+          } })
       })
     })
 

packages/app/cypress/e2e/settings.cy.ts

@@ -16,17 +16,12 @@ describe('App: Settings', () => {
     cy.visitApp()
     cy.get(SidebarSettingsLinkSelector).click()
 
-    cy.get('div[data-cy="app-header-bar"]').should('contain', 'Settings')
+    cy.contains('[data-cy="app-header-bar"]', 'Settings')
+    cy.contains('[data-cy="app-header-bar"] button', 'Log In').should('be.visible')
+
     cy.findByText('Device Settings').should('be.visible')
     cy.findByText('Project Settings').should('be.visible')
-  })
-
-  it('shows a button to log in if user is not connected', () => {
-    cy.startAppServer('e2e')
-    cy.visitApp()
-    cy.get(SidebarSettingsLinkSelector).click()
-    cy.findByText('Project Settings').click()
-    cy.get('button').contains('Log In')
+    cy.findByText('Dashboard Settings').should('be.visible')
   })
 
   describe('Cloud Settings', () => {
@@ -406,7 +401,7 @@ describe('App: Settings', () => {
 })
 
 describe('App: Settings without cloud', () => {
-  it('the projectId section shows a prompt to connect when there is no projectId', () => {
+  it('the projectId section shows a prompt to log in when there is no projectId, and uses correct UTM params', () => {
     cy.scaffoldProject('simple-ct')
     cy.openProject('simple-ct')
     cy.startAppServer('component')
@@ -415,7 +410,21 @@ describe('App: Settings without cloud', () => {
     cy.get(SidebarSettingsLinkSelector).click()
     cy.findByText('Dashboard Settings').click()
     cy.findByText('Project ID').should('exist')
-    cy.contains('button', 'Log in to the Cypress Dashboard').should('be.visible')
+    cy.withCtx((ctx, o) => {
+      o.sinon.spy(ctx._apis.authApi, 'logIn')
+    })
+
+    cy.contains('button', 'Log in to the Cypress Dashboard').click()
+    cy.findByRole('dialog', { name: 'Log in to Cypress' }).within(() => {
+      cy.contains('button', 'Log In').click()
+    })
+
+    cy.withCtx((ctx, o) => {
+      // validate utmSource
+      expect((ctx._apis.authApi.logIn as SinonStub).lastCall.args[1]).to.eq('Binary: App')
+      // validate utmMedium
+      expect((ctx._apis.authApi.logIn as SinonStub).lastCall.args[2]).to.eq('Settings Tab')
+    })
   })
 
   it('have returned browsers', () => {

packages/app/cypress/e2e/specs_list_latest_runs.cy.ts

@@ -17,6 +17,37 @@ function averageDurationSelector (specFileName: string) {
   return `${specRowSelector(specFileName)} [data-cy="average-duration"]`
 }
 
+function makeTestingCloudLink (status: string) {
+  return `https://google.com?utm_medium=Specs+Latest+Runs+Dots&utm_campaign=${status.toUpperCase()}&utm_source=Binary%3A+App`
+}
+
+function assertCorrectRunsLink (specFileName: string, status: string) {
+  // we avoid the full `cy.validateExternalLink` here because that command
+  // clicks the link, which focuses the link causing tooltips to appear,
+  // which produces problems elsewhere testing tooltip behavior
+  cy.findByRole('link', { name: specFileName })
+  .should('have.attr', 'href', makeTestingCloudLink(status))
+  .should('have.attr', 'data-cy', 'external') // to confirm the ExternalLink component is used
+}
+
+function validateTooltip (status: string) {
+  cy.validateExternalLink({
+    // TODO: (#23778) This name is so long because the entire tooltip is wrapped in a link,
+    // we can make this more accessible by having the name of the link describe the destination
+    // (which is currently not described) and keeping the other content separate.
+    name: `accounts_new.spec.js ${status} 4 months ago 2:23 - 2:39 skipped pending passed failed`,
+    // the main thing about testing this link is that is gets composed with the expected UTM params
+    href: makeTestingCloudLink(status),
+  })
+  .should('contain.text', 'accounts_new.spec.js')
+  .and('contain.text', '4 months ago')
+  .and('contain.text', '2:23 - 2:39')
+  .and('contain.text', 'skipped 0')
+  .and('contain.text', 'pending 1-2')
+  .and('contain.text', `passed 22-23`)
+  .and('contain.text', 'failed 1-2')
+}
+
 function specShouldShow (specFileName: string, runDotsClasses: string[], latestRunStatus: CloudRunStatus|'PLACEHOLDER') {
   const latestStatusSpinning = latestRunStatus === 'RUNNING'
 
@@ -31,10 +62,11 @@ function specShouldShow (specFileName: string, runDotsClasses: string[], latestR
   .should(`${latestStatusSpinning ? '' : 'not.'}have.class`, 'animate-spin')
   .and('have.attr', 'data-cy-run-status', latestRunStatus)
 
-  // TODO: add link verification
-  // if (latestRunStatus !== 'PLACEHOLDER') {
-  //   cy.get(`${specRowSelector(specFileName)} [data-cy="run-status-dots"]`).validateExternalLink('https://google.com')
-  // }
+  if (runDotsClasses?.length) {
+    assertCorrectRunsLink(`${specFileName} test results`, latestRunStatus)
+  } else {
+    cy.findByRole('link', { name: `${specFileName} test results` }).should('not.exist')
+  }
 }
 
 function simulateRunData () {
@@ -330,7 +362,9 @@ describe('App/Cloud Integration - Latest runs and Average duration', { viewportW
       specShouldShow('accounts_new.spec.js', ['gray-300', 'gray-300', 'jade-400'], 'RUNNING')
       cy.get(dotSelector('accounts_new.spec.js', 'latest')).trigger('mouseenter')
       cy.get('.v-popper__popper--shown').should('exist')
-      // TODO: verify the contents of the tooltip
+
+      validateTooltip('Running')
+
       cy.get(dotSelector('accounts_new.spec.js', 'latest')).trigger('mouseleave')
       cy.get(averageDurationSelector('accounts_new.spec.js')).contains('2:03')
     })
@@ -601,7 +635,8 @@ describe('App/Cloud Integration - Latest runs and Average duration', { viewportW
       specShouldShow('accounts_list.spec.js', ['orange-400', 'gray-300', 'red-400'], 'PASSED')
       cy.get(dotSelector('accounts_new.spec.js', 'latest')).trigger('mouseenter')
       cy.get('.v-popper__popper--shown').should('exist')
-      // TODO: verify the contents of the tooltip
+
+      validateTooltip('Passed')
       cy.get(dotSelector('accounts_new.spec.js', 'latest')).trigger('mouseleave')
       cy.get(averageDurationSelector('accounts_list.spec.js')).contains('0:12')
 
@@ -611,7 +646,8 @@ describe('App/Cloud Integration - Latest runs and Average duration', { viewportW
       specShouldShow('accounts_list.spec.js', ['orange-400', 'gray-300', 'red-400'], 'PASSED')
       cy.get(dotSelector('accounts_new.spec.js', 'latest')).trigger('mouseenter')
       cy.get('.v-popper__popper--shown').should('exist')
-      // TODO: verify the contents of the tooltip
+
+      validateTooltip('Passed')
       cy.get(dotSelector('accounts_new.spec.js', 'latest')).trigger('mouseleave')
       cy.get(averageDurationSelector('accounts_list.spec.js')).contains('0:12')
     })

packages/app/cypress/e2e/top-nav.cy.ts

@@ -407,6 +407,7 @@ describe('App Top Nav Workflows', () => {
 
       const mockLogInActionsForUser = (user) => {
         cy.withCtx(async (ctx, options) => {
+          ctx.coreData.app.browserStatus = 'open'
           options.sinon.stub(ctx._apis.electronApi, 'isMainWindowFocused').returns(false)
           options.sinon.stub(ctx._apis.authApi, 'logIn').callsFake(async (onMessage) => {
             setTimeout(() => {
@@ -455,6 +456,13 @@ describe('App Top Nav Workflows', () => {
 
           mockLogInActionsForUser(mockUser)
           logIn({ expectedNextStepText: 'Connect project', displayName: mockUser.name })
+          cy.withCtx((ctx, o) => {
+            // validate utmSource
+            expect((ctx._apis.authApi.logIn as SinonStub).lastCall.args[1]).to.eq('Binary: App')
+            // validate utmMedium
+            expect((ctx._apis.authApi.logIn as SinonStub).lastCall.args[2]).to.eq('Nav')
+          })
+
           cy.findByRole('dialog', { name: 'Create project' }).should('be.visible')
         })
       })

packages/app/src/layouts/default.vue

@@ -49,10 +49,13 @@
           <component :is="Component" />
         </transition>
       </router-view>
+      <!-- "Nav" is the correct UTM medium below because
+        this is only opened by event emitted from the header bar-->
       <CloudConnectModals
         v-if="showConnectDialog && cloudModalQuery.data.value"
         :show="showConnectDialog"
         :gql="cloudModalQuery.data.value"
+        utm-medium="Nav"
         @cancel="showConnectDialog = false"
         @success="showConnectDialog = false"
       />

packages/app/src/runs/CloudConnectButton.cy.tsx

@@ -9,7 +9,7 @@ describe('<CloudConnectButton />', () => {
         result.cloudViewer = null
       },
       render (gqlVal) {
-        return <div class="h-screen"><CloudConnectButton gql={gqlVal} /></div>
+        return <div class="h-screen"><CloudConnectButton utmMedium="testing" gql={gqlVal} /></div>
       },
     })
 
@@ -56,7 +56,7 @@ describe('<CloudConnectButton />', () => {
         result.cloudViewer = cloudViewer
       },
       render (gqlVal) {
-        return <div class="h-screen"><CloudConnectButton gql={gqlVal} /></div>
+        return <div class="h-screen"><CloudConnectButton utmMedium="testing" gql={gqlVal} /></div>
       },
     })
 
@@ -69,7 +69,7 @@ describe('<CloudConnectButton />', () => {
         result.cloudViewer = cloudViewer
       },
       render (gqlVal) {
-        return <div class="h-screen"><CloudConnectButton gql={gqlVal} /></div>
+        return <div class="h-screen"><CloudConnectButton utmMedium="testing" gql={gqlVal} /></div>
       },
     })
 

packages/app/src/runs/CloudConnectButton.vue

@@ -10,14 +10,15 @@
   <LoginModal
     v-model="isLoginOpen"
     :gql="props.gql"
-    utm-medium="Runs Tab"
+    :utm-medium="props.utmMedium"
     :show-connect-button-after-login="!cloudProjectId"
     @connect-project="isProjectConnectOpen = true"
   />
   <CloudConnectModals
     v-if="isProjectConnectOpen"
     :show="isProjectConnectOpen"
     :gql="props.gql"
+    :utm-medium="props.utmMedium"
     @cancel="isProjectConnectOpen = false"
     @success="isProjectConnectOpen = false; emit('success')"
   />
@@ -54,6 +55,7 @@ const emit = defineEmits<{
 const props = defineProps<{
   gql: CloudConnectButtonFragment
   class?: string
+  utmMedium: string
 }>()
 
 const isLoginOpen = ref(false)

packages/app/src/runs/RunsConnect.vue

@@ -20,6 +20,7 @@
     <CloudConnectButton
       :gql="props.gql"
       class="mx-auto mt-40px"
+      utm-medium="Runs Tab"
       @success="emit('success')"
     />
   </div>

packages/app/src/runs/RunsErrorRenderer.vue

@@ -62,6 +62,7 @@
     v-if="showConnectDialog"
     :show="showConnectDialog"
     :gql="props.gql"
+    utm-medium="Runs Tab"
     @cancel="showConnectDialog = false"
     @success="showConnectDialog = false"
   />

packages/app/src/runs/modals/CloudConnectModals.spec.tsx

@@ -38,7 +38,7 @@ describe('<CloudConnectModals />', () => {
       },
       render (gql) {
         return (<div class="h-screen">
-          <CloudConnectModals gql={gql}/>
+          <CloudConnectModals utmMedium="testing" gql={gql}/>
         </div>)
       },
     })

packages/app/src/runs/modals/CloudConnectModals.vue

@@ -11,6 +11,7 @@
     v-else-if="props.gql.cloudViewer?.organizations?.nodes.length ?? 0 > 0"
     :gql="props.gql"
     show
+    :utm-medium="props.utmMedium"
     @update-project-id-failed="showManualUpdate"
     @success="emit('success')"
     @cancel="emit('cancel')"
@@ -71,6 +72,7 @@ const emit = defineEmits<{
 
 const props = defineProps<{
   gql: CloudConnectModalsFragment
+  utmMedium: string
 }>()
 
 const newProjectId = ref('')

packages/app/src/runs/modals/SelectCloudProjectModal.cy.tsx

@@ -25,7 +25,7 @@ describe('<SelectCloudProjectModal />', () => {
       },
       render (gql) {
         return (<div class="h-screen">
-          <SelectCloudProjectModal gql={gql}/>
+          <SelectCloudProjectModal utmMedium="test" gql={gql}/>
         </div>)
       },
     })
@@ -144,7 +144,7 @@ describe('<SelectCloudProjectModal />', () => {
         cy.get('@createMutation').should('have.been.calledOnceWith', {
           campaign: 'Create project',
           cohort: '',
-          medium: 'Specs Create Project Banner',
+          medium: 'test',
           name: 'Test Project',
           orgId: '1',
           public: false,

packages/app/src/runs/modals/SelectCloudProjectModal.vue

@@ -274,6 +274,7 @@ mutation SelectCloudProjectModal_CreateCloudProject( $name: String!, $orgId: ID!
 
 const props = defineProps<{
   gql: SelectCloudProjectModalFragment
+  utmMedium: string
 }>()
 
 const emit = defineEmits<{
@@ -346,7 +347,7 @@ async function createOrConnectProject () {
       public: projectAccess.value === 'public',
       campaign: 'Create project',
       cohort: '',
-      medium: 'Specs Create Project Banner',
+      medium: props.utmMedium,
       source: getUtmSource(),
     })
 

packages/app/src/settings/project/ProjectId.vue

@@ -34,6 +34,7 @@
     <CloudConnectButton
       v-else
       :gql="props.gql"
+      utm-medium="Settings Tab"
     />
   </SettingsSection>
 </template>

packages/app/src/specs/RunStatusDots.vue

@@ -43,6 +43,10 @@
               class="ml-4px"
             />
           </div>
+          <span
+            v-if="latestRun"
+            class="sr-only"
+          >{{ props.specFileName }}{{ props.specFileExtension }} test results</span>
         </div>
       </component>
       <template

packages/app/src/specs/SpecsList.vue

@@ -182,6 +182,7 @@
   <CloudConnectModals
     v-if="isProjectConnectOpen"
     :gql="props.gql"
+    :utm-medium="loginUtmMedium"
     @cancel="isProjectConnectOpen = false"
     @success="refreshPage"
   />