Adds twitch support

Resolves brave#13139

Auditors:

Test Plan:

app/browser/api/ledger.js

@@ -315,6 +315,7 @@ const getPublisherData = (result, scorekeeper) => {
     verified: result.options.verified || false,
     exclude: result.options.exclude || false,
     publisherKey: result.publisherKey,
+    providerName: result.providerName,
     siteName: result.publisherKey,
     views: result.visits,
     duration: duration,
@@ -1387,7 +1388,7 @@ const roundtrip = (params, options, callback) => {
     : typeof params.server !== 'undefined' ? params.server
       : typeof options.server === 'string' ? urlParse(options.server) : options.server
   const binaryP = options.binaryP
-  const rawP = binaryP || options.rawP
+  const rawP = binaryP || options.rawP || options.scrapeP
 
   if (!params.method) params.method = 'GET'
   parts = underscore.extend(underscore.pick(parts, ['protocol', 'hostname', 'port']),
@@ -2399,10 +2400,15 @@ const onMediaRequest = (state, xhr, type, tabId) => {
 
   const parsed = ledgerUtil.getMediaData(xhr, type)
   const mediaId = ledgerUtil.getMediaId(parsed, type)
+
+  if (mediaId == null) {
+    return state
+  }
+
   const mediaKey = ledgerUtil.getMediaKey(mediaId, type)
-  let duration = ledgerUtil.getMediaDuration(parsed, type)
+  let duration = ledgerUtil.getMediaDuration(state, parsed, mediaKey, type)
 
-  if (mediaId == null || duration == null || mediaKey == null) {
+  if (duration == null || mediaKey == null) {
     return state
   }
 
@@ -2422,9 +2428,14 @@ const onMediaRequest = (state, xhr, type, tabId) => {
     currentMediaKey = mediaKey
   }
 
+  const stateData = ledgerUtil.generateMediaCacheData(parsed, type)
   const cache = ledgerVideoCache.getDataByVideoId(state, mediaKey)
 
   if (!cache.isEmpty()) {
+    if (!stateData.isEmpty()) {
+      state = ledgerVideoCache.mergeCacheByVideoId(state, mediaKey, stateData)
+    }
+
     const publisherKey = cache.get('publisher')
     const publisher = ledgerState.getPublisher(state, publisherKey)
     if (!publisher.isEmpty() && publisher.has('providerName')) {
@@ -2436,6 +2447,10 @@ const onMediaRequest = (state, xhr, type, tabId) => {
     }
   }
 
+  if (!stateData.isEmpty()) {
+    state = ledgerVideoCache.setCacheByVideoId(state, mediaKey, stateData)
+  }
+
   const options = underscore.extend({roundtrip: module.exports.roundtrip}, clientOptions)
   const mediaProps = {
     mediaId,
@@ -2513,7 +2528,7 @@ const onMediaPublisher = (state, mediaKey, response, duration, revisited) => {
     .set('publisher', publisherKey)
 
   // Add to cache
-  state = ledgerVideoCache.setCacheByVideoId(state, mediaKey, cacheObject)
+  state = ledgerVideoCache.mergeCacheByVideoId(state, mediaKey, cacheObject)
 
   state = module.exports.saveVisit(state, publisherKey, {
     duration,

app/common/cache/ledgerVideoCache.js

@@ -33,7 +33,24 @@ const setCacheByVideoId = (state, key, data) => {
   return state.setIn(['cache', 'ledgerVideos', key], data)
 }
 
+const mergeCacheByVideoId = (state, key, data) => {
+  state = validateState(state)
+
+  if (key == null || data == null) {
+    return state
+  }
+
+  data = makeImmutable(data)
+
+  if (data.isEmpty()) {
+    return state
+  }
+
+  return state.mergeIn(['cache', 'ledgerVideos', key], data)
+}
+
 module.exports = {
   getDataByVideoId,
-  setCacheByVideoId
+  setCacheByVideoId,
+  mergeCacheByVideoId
 }

app/common/constants/ledgerMediaProviders.js

@@ -3,7 +3,8 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 const providers = {
-  YOUTUBE: 'youtube'
+  YOUTUBE: 'youtube',
+  TWITCH: 'twitch'
 }
 
 module.exports = providers

app/common/lib/ledgerUtil.js

@@ -13,6 +13,7 @@ const queryString = require('querystring')
 // State
 const siteSettingsState = require('../state/siteSettingsState')
 const ledgerState = require('../state/ledgerState')
+const ledgerVideoCache = require('../cache/ledgerVideoCache')
 
 // Constants
 const settings = require('../../../js/constants/settings')
@@ -23,6 +24,7 @@ const {responseHasContent} = require('./httpUtil')
 const urlUtil = require('../../../js/lib/urlutil')
 const getSetting = require('../../../js/settings').getSetting
 const urlParse = require('../urlParse')
+
 /**
  * Is page an actual page being viewed by the user? (not an error page, etc)
  * If the page is invalid, we don't want to collect usage info.
@@ -221,6 +223,26 @@ const getMediaId = (data, type) => {
         id = data.docid
         break
       }
+    case ledgerMediaProviders.TWITCH:
+      {
+        if (
+          (
+            data.event === 'minute-watched' ||
+            data.event === 'video-play' ||
+            data.event === 'player_click_playpause' ||
+            data.event === 'vod_seek'
+          ) &&
+          data.properties
+        ) {
+          id = data.properties.channel
+          let vod = data.properties.vod
+
+          if (vod) {
+            vod = vod.replace('v', '')
+            id += `_vod_${vod}`
+          }
+        }
+      }
   }
 
   return id
@@ -241,35 +263,181 @@ const getMediaData = (xhr, type) => {
     return result
   }
 
+  const parsedUrl = urlParse(xhr)
+  const query = parsedUrl && parsedUrl.query
+
+  if (!parsedUrl || !query) {
+    return null
+  }
+
   switch (type) {
     case ledgerMediaProviders.YOUTUBE:
       {
-        const parsedUrl = urlParse(xhr)
-        let query = null
+        result = queryString.parse(parsedUrl.query)
+        break
+      }
+    case ledgerMediaProviders.TWITCH:
+      {
+        result = queryString.parse(parsedUrl.query)
+        if (!result.data) {
+          result = null
+          break
+        }
+
+        let obj = Buffer.from(result.data, 'base64').toString('utf8')
+        if (obj == null) {
+          result = null
+          break
+        }
 
-        if (parsedUrl && parsedUrl.query) {
-          query = queryString.parse(parsedUrl.query)
+        try {
+          result = JSON.parse(obj)
+        } catch (error) {
+          result = null
+          console.error(error.toString())
         }
-        result = query
         break
       }
   }
 
   return result
 }
 
-const getMediaDuration = (data, type) => {
+const getMediaDuration = (state, data, mediaKey, type) => {
   let duration = 0
+
+  if (data == null) {
+    return duration
+  }
+
   switch (type) {
-    case ledgerMediaProviders.YOUTUBE: {
-      duration = getYouTubeDuration(data)
-      break
-    }
+    case ledgerMediaProviders.YOUTUBE:
+      {
+        duration = getYouTubeDuration(data)
+        break
+      }
+    case ledgerMediaProviders.TWITCH:
+      {
+        duration = getTwitchDuration(state, data, mediaKey)
+        break
+      }
   }
 
   return duration
 }
 
+const generateMediaCacheData = (parsed, type) => {
+  let data = Immutable.Map()
+
+  if (parsed == null) {
+    return data
+  }
+
+  switch (type) {
+    case ledgerMediaProviders.TWITCH:
+      {
+        data = generateTwitchCacheData(parsed)
+        break
+      }
+  }
+
+  return data
+}
+
+const generateTwitchCacheData = (parsed) => {
+  if (parsed == null) {
+    return Immutable.Map()
+  }
+
+  if (parsed.properties) {
+    return Immutable.fromJS({
+      event: parsed.event,
+      time: parsed.properties.time
+    })
+  }
+
+  return Immutable.fromJS({
+    event: parsed.event
+  })
+}
+
+const getMediaFavicon = (providerName) => {
+  let image = null
+
+  if (!providerName) {
+    return image
+  }
+
+  providerName = providerName.toLowerCase()
+
+  switch (providerName) {
+    case ledgerMediaProviders.YOUTUBE:
+      {
+        image = require('../../../img/mediaProviders/youtube.png')
+        break
+      }
+    case ledgerMediaProviders.TWITCH:
+      {
+        image = require('../../../img/mediaProviders/twitch.svg')
+        break
+      }
+  }
+
+  return image
+}
+
+const getTwitchDuration = (state, data, mediaKey) => {
+  if (data == null || mediaKey == null) {
+    return 0
+  }
+
+  const previousData = ledgerVideoCache.getDataByVideoId(state, mediaKey)
+
+  if (previousData.isEmpty() && data.event === 'video-play') {
+    return milliseconds.second * 10
+  }
+
+  if (!data.properties) {
+    return 0
+  }
+
+  const oldMedia = ledgerVideoCache.getDataByVideoId(state, mediaKey)
+  let time = 0
+  const currentTime = parseFloat(data.properties.time)
+  const oldTime = parseFloat(previousData.get('time'))
+
+  if (
+    data.event === 'player_click_playpause' &&
+    oldMedia.get('event') !== 'player_click_playpause'
+  ) {
+    // User paused a video
+    time = currentTime - oldTime
+  } else if (previousData.get('event') === 'video-play') {
+    // From video play event to x event
+    time = currentTime - oldTime - 10
+  } else if (data.event === 'minute-watched') {
+    // Minute watched event
+    time = currentTime - oldTime
+  }
+
+  if (isNaN(time)) {
+    return 0
+  }
+
+  if (time < 0) {
+    return 0
+  }
+
+  if (time > 120) {
+    time = 120 // 2 minutes
+  }
+
+  // we get seconds back, so we need to convert it into ms
+  time = time * 1000
+
+  return Math.floor(time)
+}
+
 const getYouTubeDuration = (data) => {
   let time = 0
 
@@ -294,7 +462,7 @@ const getYouTubeDuration = (data) => {
   return parseInt(time)
 }
 
-const getMediaProvider = (url) => {
+const getMediaProvider = (url, firstPartyUrl, referrer) => {
   let provider = null
 
   if (url == null) {
@@ -303,7 +471,18 @@ const getMediaProvider = (url) => {
 
   // Youtube
   if (url.startsWith('https://www.youtube.com/api/stats/watchtime?')) {
-    provider = ledgerMediaProviders.YOUTUBE
+    return ledgerMediaProviders.YOUTUBE
+  }
+
+  // Twitch
+  if (
+    (
+      (firstPartyUrl && firstPartyUrl.startsWith('https://www.twitch.tv')) ||
+      (referrer && referrer.startsWith('https://player.twitch.tv'))
+    ) &&
+    url.startsWith('https://api.mixpanel.com')
+  ) {
+    return ledgerMediaProviders.TWITCH
   }
 
   return provider
@@ -339,14 +518,18 @@ const getMethods = () => {
     getMediaData,
     getMediaKey,
     milliseconds,
-    defaultMonthlyAmounts
+    defaultMonthlyAmounts,
+    getMediaFavicon,
+    generateMediaCacheData
   }
 
   let privateMethods = {}
 
   if (process.env.NODE_ENV === 'test') {
     privateMethods = {
-      getYouTubeDuration
+      getYouTubeDuration,
+      getTwitchDuration,
+      generateTwitchCacheData
     }
   }