Fixes twitch, updated to the new data

Resolves brave#13597

Auditors:

Test Plan:

app/browser/api/ledger.js

@@ -2584,12 +2584,17 @@ const deleteSynopsis = () => {
 }
 
 let currentMediaKey = null
-const onMediaRequest = (state, xhr, type, tabId) => {
+const onMediaRequest = (state, xhr, type, details) => {
   if (!xhr || type == null) {
     return state
   }
 
-  const parsed = ledgerUtil.getMediaData(xhr, type)
+  const tabId = details.get('tabId')
+  const parsed = ledgerUtil.getMediaData(xhr, type, details)
+  if (parsed == null) {
+    return state
+  }
+
   const mediaId = ledgerUtil.getMediaId(parsed, type)
 
   if (clientOptions.loggingP) {

app/browser/reducers/ledgerReducer.js

@@ -418,7 +418,7 @@ const ledgerReducer = (state, action, immutableAction) => {
       }
     case appConstants.APP_ON_LEDGER_MEDIA_DATA:
       {
-        state = ledgerApi.onMediaRequest(state, action.get('url'), action.get('type'), action.get('tabId'))
+        state = ledgerApi.onMediaRequest(state, action.get('url'), action.get('type'), action.get('details'))
         break
       }
     case appConstants.APP_ON_PRUNE_SYNOPSIS:

app/common/constants/twitchEvents.js

@@ -7,7 +7,7 @@ const events = {
   BUFFER_REFILL: 'buffer-refill',
   MINUTE_WATCHED: 'minute-watched',
   PLAY_MANIFEST: 'video_play_master_manifest',
-  PLAY_PAUSE: 'player_click_playpause',
+  PLAY_PAUSE: 'video_pause',
   SEEK: 'player_click_vod_seek',
   START: 'video-play',
   END: 'video_end',

app/common/lib/ledgerUtil.js

@@ -252,7 +252,7 @@ const getMediaKey = (id, type) => {
   return `${type.toLowerCase()}_${id}`
 }
 
-const getMediaData = (xhr, type) => {
+const getMediaData = (xhr, type, details) => {
   let result = null
 
   if (xhr == null || type == null) {
@@ -262,36 +262,58 @@ const getMediaData = (xhr, type) => {
   const parsedUrl = urlParse(xhr)
   const query = parsedUrl && parsedUrl.query
 
-  if (!parsedUrl || !query) {
+  if (!parsedUrl) {
     return null
   }
 
   switch (type) {
     case ledgerMediaProviders.YOUTUBE:
       {
-        result = queryString.parse(parsedUrl.query)
+        if (!query) {
+          return null
+        }
+
+        result = queryString.parse(query)
         break
       }
     case ledgerMediaProviders.TWITCH:
       {
-        result = queryString.parse(parsedUrl.query)
-        if (!result.data) {
+        const data = details.getIn(['uploadData', 0, 'bytes'])
+        if (!data) {
           result = null
           break
         }
 
-        let obj = Buffer.from(result.data, 'base64').toString('utf8')
+        let params = Buffer.from(data).toString('utf8') || ''
+        const paramQuery = queryString.parse(params)
+
+        if (!paramQuery || !paramQuery.data) {
+          result = null
+          break
+        }
+
+        let obj = Buffer.from(paramQuery.data, 'base64').toString('utf8')
         if (obj == null) {
           result = null
           break
         }
 
+        let parsed
         try {
-          result = JSON.parse(obj)
+          parsed = JSON.parse(obj)
         } catch (error) {
           result = null
-          console.error(error.toString())
+          console.error(error.toString(), obj)
+          break
         }
+
+        if (!Array.isArray(parsed)) {
+          result = null
+          break
+        }
+
+        result = parsed[0]
+
         break
       }
   }
@@ -425,6 +447,15 @@ const getTwitchDuration = (state, data, mediaKey) => {
   }
 
   const previousData = ledgerVideoCache.getDataByVideoId(state, mediaKey)
+
+  // remove duplicate events
+  if (
+    previousData.get('event') === data.event &&
+    previousData.get('time') === data.properties.time
+  ) {
+    return null
+  }
+
   const oldEvent = previousData.get('event')
   const twitchMinimumSeconds = 10
 
@@ -524,7 +555,10 @@ const getMediaProvider = (url, firstPartyUrl, referrer) => {
       (firstPartyUrl && firstPartyUrl.startsWith('https://www.twitch.tv/')) ||
       (referrer && referrer.startsWith('https://player.twitch.tv/'))
     ) &&
-    url.startsWith('https://api.mixpanel.com/')
+    (
+      url.includes('.ttvnw.net/v1/segment/') ||
+      url.includes('https://ttvnw.net/v1/segment/')
+    )
   ) {
     return ledgerMediaProviders.TWITCH
   }

app/filtering.js

@@ -126,7 +126,7 @@ function registerForBeforeRequest (session, partition) {
       // Ledger media
       const provider = ledgerUtil.getMediaProvider(url, firstPartyUrl, details.referrer)
       if (provider) {
-        appActions.onLedgerMediaData(url, provider, details.tabId)
+        appActions.onLedgerMediaData(url, provider, details)
       }
     }
 

js/actions/appActions.js

@@ -1897,12 +1897,12 @@ const appActions = {
     })
   },
 
-  onLedgerMediaData: function (url, type, tabId) {
+  onLedgerMediaData: function (url, type, details) {
     dispatch({
       actionType: appConstants.APP_ON_LEDGER_MEDIA_DATA,
       url,
       type,
-      tabId
+      details
     })
   },
 

test/unit/app/browser/api/ledgerTest.js

@@ -614,21 +614,21 @@ describe('ledger api unit tests', function () {
         })
         it('does nothing if tab is private', function () {
           const xhr2 = 'https://www.youtube.com/api/stats/watchtime?docid=kLiLOkzLetE&st=20.338&et=21.339'
-          ledgerApi.onMediaRequest(cacheAppState, xhr2, ledgerMediaProviders.YOUTUBE, 1)
+          ledgerApi.onMediaRequest(cacheAppState, xhr2, ledgerMediaProviders.YOUTUBE, Immutable.fromJS({tabId: 1}))
           assert(mediaSpy.notCalled)
           assert(saveVisitSpy.notCalled)
         })
       })
 
       it('set currentMediaKey when it is different than saved', function () {
-        ledgerApi.onMediaRequest(defaultAppState, xhr, ledgerMediaProviders.YOUTUBE, 1)
+        ledgerApi.onMediaRequest(defaultAppState, xhr, ledgerMediaProviders.YOUTUBE, Immutable.fromJS({tabId: 1}))
         assert.equal(ledgerApi.getCurrentMediaKey(), videoId)
         assert(mediaSpy.calledOnce)
         assert(saveVisitSpy.notCalled)
       })
 
       it('get data from cache, if we have publisher in synopsis', function () {
-        ledgerApi.onMediaRequest(cacheAppState, xhr, ledgerMediaProviders.YOUTUBE, 1)
+        ledgerApi.onMediaRequest(cacheAppState, xhr, ledgerMediaProviders.YOUTUBE, Immutable.fromJS({tabId: 1}))
         assert(mediaSpy.notCalled)
         assert(saveVisitSpy.withArgs(cacheAppState, publisherKey, {
           duration: 10001,
@@ -641,14 +641,14 @@ describe('ledger api unit tests', function () {
         const state = defaultAppState.setIn(['cache', 'ledgerVideos', videoId], Immutable.fromJS({
           publisher: publisherKey
         }))
-        ledgerApi.onMediaRequest(state, xhr, ledgerMediaProviders.YOUTUBE, 1)
+        ledgerApi.onMediaRequest(state, xhr, ledgerMediaProviders.YOUTUBE, Immutable.fromJS({tabId: 1}))
         assert(mediaSpy.calledOnce)
         assert(saveVisitSpy.notCalled)
       })
 
       it('revisited if visiting the same media in the same tab', function () {
         // first call, revisit false
-        ledgerApi.onMediaRequest(cacheAppState, xhr, ledgerMediaProviders.YOUTUBE, 1)
+        ledgerApi.onMediaRequest(cacheAppState, xhr, ledgerMediaProviders.YOUTUBE, Immutable.fromJS({tabId: 1}))
         assert.equal(ledgerApi.getCurrentMediaKey(), videoId)
         assert(saveVisitSpy.withArgs(cacheAppState, publisherKey, {
           duration: 10001,
@@ -657,7 +657,7 @@ describe('ledger api unit tests', function () {
         }).calledOnce)
 
         // second call, revisit true
-        ledgerApi.onMediaRequest(cacheAppState, xhr, ledgerMediaProviders.YOUTUBE, 1)
+        ledgerApi.onMediaRequest(cacheAppState, xhr, ledgerMediaProviders.YOUTUBE, Immutable.fromJS({tabId: 1}))
         assert(mediaSpy.notCalled)
         assert(saveVisitSpy.withArgs(cacheAppState, publisherKey, {
           duration: 10001,
@@ -669,7 +669,7 @@ describe('ledger api unit tests', function () {
       it('revisited if visiting media in the background tab', function () {
         // first call, revisit false
         ledgerApi.setCurrentMediaKey('11')
-        ledgerApi.onMediaRequest(cacheAppState, xhr, ledgerMediaProviders.YOUTUBE, 10)
+        ledgerApi.onMediaRequest(cacheAppState, xhr, ledgerMediaProviders.YOUTUBE, Immutable.fromJS({tabId: 10}))
         assert.equal(ledgerApi.getCurrentMediaKey(), '11')
         assert(saveVisitSpy.withArgs(cacheAppState, publisherKey, {
           duration: 10001,
@@ -689,7 +689,7 @@ describe('ledger api unit tests', function () {
             }
           }))
 
-        ledgerApi.onMediaRequest(badState, xhr, ledgerMediaProviders.YOUTUBE, 10)
+        ledgerApi.onMediaRequest(badState, xhr, ledgerMediaProviders.YOUTUBE, Immutable.fromJS({tabId: 10}))
         assert(mediaSpy.calledOnce)
         assert(saveVisitSpy.notCalled)
       })

test/unit/app/common/lib/ledgerUtilTest.js

@@ -516,23 +516,55 @@ describe('ledgerUtil unit test', function () {
     })
 
     describe('Twitch', function () {
+      const url = 'https://video-edge-f0f586.sjc01.hls.ttvnw.net/v1/segment/CuDNI7xCy5CGJ8g7G3thdHT26OW_DhnEuVw0tRGN-DKhJxrRTeGe...'
+
       it('null case', function () {
         const result = ledgerUtil.getMediaData(null, ledgerMediaProviders.TWITCH)
         assert.equal(result, null)
       })
 
+      it('uploadData is missing', function () {
+        const result = ledgerUtil.getMediaData(url, ledgerMediaProviders.TWITCH, Immutable.fromJS({
+          firstPartyUrl: 'https://www.twitch.tv/videos/241926348'
+        }))
+        assert.equal(result, null)
+      })
+
+      it('bytes is missing', function () {
+        const result = ledgerUtil.getMediaData(url, ledgerMediaProviders.TWITCH, Immutable.fromJS({
+          firstPartyUrl: 'https://www.twitch.tv/videos/241926348',
+          uploadData: []
+        }))
+        assert.equal(result, null)
+      })
+
       it('data is missing', function () {
-        const result = ledgerUtil.getMediaData('https://api.mixpanel.com', ledgerMediaProviders.TWITCH)
+        const result = ledgerUtil.getMediaData(url, ledgerMediaProviders.TWITCH, Immutable.fromJS({
+          firstPartyUrl: 'https://www.twitch.tv/videos/241926348',
+          uploadData: [{
+            bytes: new Uint8Array([116, 101, 115, 116])
+          }]
+        }))
         assert.equal(result, null)
       })
 
       it('data is empty string', function () {
-        const result = ledgerUtil.getMediaData('https://api.mixpanel.com?data=', ledgerMediaProviders.TWITCH)
+        const result = ledgerUtil.getMediaData(url, ledgerMediaProviders.TWITCH, Immutable.fromJS({
+          firstPartyUrl: 'https://www.twitch.tv/videos/241926348',
+          uploadData: [{
+            bytes: new Uint8Array([100, 97, 116, 97, 61])
+          }]
+        }))
         assert.equal(result, null)
       })
 
       it('obj is parsed correctly', function () {
-        const result = ledgerUtil.getMediaData('https://api.mixpanel.com?data=eyJldmVudCI6Im1pbnV0ZS13YXRjaGVkIiwicHJvcGVydGllcyI6eyJjaGFubmVsIjoidHcifX0=', ledgerMediaProviders.TWITCH)
+        const result = ledgerUtil.getMediaData(url, ledgerMediaProviders.TWITCH, Immutable.fromJS({
+          firstPartyUrl: 'https://www.twitch.tv/videos/241926348',
+          uploadData: [{
+            bytes: new Uint8Array([100, 97, 116, 97, 61, 87, 51, 115, 105, 90, 88, 90, 108, 98, 110, 81, 105, 79, 105, 74, 116, 97, 87, 53, 49, 100, 71, 85, 116, 100, 50, 70, 48, 89, 50, 104, 108, 90, 67, 73, 115, 73, 110, 66, 121, 98, 51, 66, 108, 99, 110, 82, 112, 90, 88, 77, 105, 79, 110, 115, 105, 89, 50, 104, 104, 98, 109, 53, 108, 98, 67, 73, 54, 73, 110, 82, 51, 73, 110, 49, 57, 88, 81, 61, 61])
+          }]
+        }))
         assert.deepEqual(result, {
           event: twitchEvents.MINUTE_WATCHED,
           properties: {
@@ -584,21 +616,21 @@ describe('ledgerUtil unit test', function () {
 
     describe('twitch', function () {
       it('we only have url', function () {
-        const result = ledgerUtil.getMediaProvider('https://api.mixpanel.com/?data=lll')
+        const result = ledgerUtil.getMediaProvider('https://ttvnw.net/v1/segment/sdfsdfsdfdsf')
         assert.equal(result, null)
       })
 
       it('video is on twitch.tv', function () {
         const result = ledgerUtil.getMediaProvider(
-          'https://api.mixpanel.com/?data=lll',
+          'https://ttvnw.net/v1/segment/sdfsdfsdfdsf',
           'https://www.twitch.tv/'
         )
         assert.equal(result, ledgerMediaProviders.TWITCH)
       })
 
       it('video is embeded', function () {
         const result = ledgerUtil.getMediaProvider(
-          'https://api.mixpanel.com/?data=lll',
+          'https://ttvnw.net/v1/segment/sdfsdfsdfdsf',
           'https://www.site.tv/',
           'https://player.twitch.tv/'
         )