Adds twitch support

Resolves #13139

Auditors:

Test Plan:

app/browser/api/ledger.js

@@ -315,6 +315,7 @@ const getPublisherData = (result, scorekeeper) => {
  verified: result.options.verified || false,
  exclude: result.options.exclude || false,
  publisherKey: result.publisherKey,
+ providerName: result.providerName,
  siteName: result.publisherKey,
  views: result.visits,
  duration: duration,
@@ -1387,7 +1388,7 @@ const roundtrip = (params, options, callback) => {
  : typeof params.server !== 'undefined' ? params.server
  : typeof options.server === 'string' ? urlParse(options.server) : options.server
  const binaryP = options.binaryP
- const rawP = binaryP || options.rawP
+ const rawP = binaryP || options.rawP || options.scrapeP
 
  if (!params.method) params.method = 'GET'
  parts = underscore.extend(underscore.pick(parts, ['protocol', 'hostname', 'port']),
@@ -2399,10 +2400,15 @@ const onMediaRequest = (state, xhr, type, tabId) => {
 
  const parsed = ledgerUtil.getMediaData(xhr, type)
  const mediaId = ledgerUtil.getMediaId(parsed, type)
+
+ if (mediaId == null) {
+ return state
+ }
+
  const mediaKey = ledgerUtil.getMediaKey(mediaId, type)
- let duration = ledgerUtil.getMediaDuration(parsed, type)
+ let duration = ledgerUtil.getMediaDuration(state, parsed, mediaKey, type)
 
- if (mediaId == null || duration == null || mediaKey == null) {
+ if (duration == null || mediaKey == null) {
  return state
  }
 
@@ -2422,9 +2428,14 @@ const onMediaRequest = (state, xhr, type, tabId) => {
  currentMediaKey = mediaKey
  }
 
+ const stateData = ledgerUtil.generateMediaCacheData(parsed, type)
  const cache = ledgerVideoCache.getDataByVideoId(state, mediaKey)
 
  if (!cache.isEmpty()) {
+ if (!stateData.isEmpty()) {
+ state = ledgerVideoCache.mergeCacheByVideoId(state, mediaKey, stateData)
+ }
+
  const publisherKey = cache.get('publisher')
  const publisher = ledgerState.getPublisher(state, publisherKey)
  if (!publisher.isEmpty() && publisher.has('providerName')) {
@@ -2436,6 +2447,10 @@ const onMediaRequest = (state, xhr, type, tabId) => {
  }
  }
 
+ if (!stateData.isEmpty()) {
+ state = ledgerVideoCache.setCacheByVideoId(state, mediaKey, stateData)
+ }
+
  const options = underscore.extend({roundtrip: module.exports.roundtrip}, clientOptions)
  const mediaProps = {
  mediaId,
@@ -2513,7 +2528,7 @@ const onMediaPublisher = (state, mediaKey, response, duration, revisited) => {
  .set('publisher', publisherKey)
 
  // Add to cache
- state = ledgerVideoCache.setCacheByVideoId(state, mediaKey, cacheObject)
+ state = ledgerVideoCache.mergeCacheByVideoId(state, mediaKey, cacheObject)
 
  state = module.exports.saveVisit(state, publisherKey, {
  duration,

app/common/cache/ledgerVideoCache.js

@@ -33,7 +33,24 @@ const setCacheByVideoId = (state, key, data) => {
  return state.setIn(['cache', 'ledgerVideos', key], data)
 }
 
+const mergeCacheByVideoId = (state, key, data) => {
+ state = validateState(state)
+
+ if (key == null || data == null) {
+ return state
+ }
+
+ data = makeImmutable(data)
+
+ if (data.isEmpty()) {
+ return state
+ }
+
+ return state.mergeIn(['cache', 'ledgerVideos', key], data)
+}
+
 module.exports = {
  getDataByVideoId,
- setCacheByVideoId
+ setCacheByVideoId,
+ mergeCacheByVideoId
 }

app/common/constants/ledgerMediaProviders.js

@@ -3,7 +3,8 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 const providers = {
- YOUTUBE: 'youtube'
+ YOUTUBE: 'youtube',
+ TWITCH: 'twitch'
 }
 
 module.exports = providers

app/common/lib/ledgerUtil.js

@@ -13,6 +13,7 @@ const queryString = require('querystring')
 // State
 const siteSettingsState = require('../state/siteSettingsState')
 const ledgerState = require('../state/ledgerState')
+const ledgerVideoCache = require('../cache/ledgerVideoCache')
 
 // Constants
 const settings = require('../../../js/constants/settings')
@@ -23,6 +24,7 @@ const {responseHasContent} = require('./httpUtil')
 const urlUtil = require('../../../js/lib/urlutil')
 const getSetting = require('../../../js/settings').getSetting
 const urlParse = require('../urlParse')
+
 /**
  * Is page an actual page being viewed by the user? (not an error page, etc)
  * If the page is invalid, we don't want to collect usage info.
@@ -221,6 +223,26 @@ const getMediaId = (data, type) => {
  id = data.docid
  break
  }
+ case ledgerMediaProviders.TWITCH:
+ {
+ if (
+ (
+ data.event === 'minute-watched' ||
+ data.event === 'video-play' ||
+ data.event === 'player_click_playpause' ||
+ data.event === 'vod_seek'
+ ) &&
+ data.properties
+ ) {
+ id = data.properties.channel
+ let vod = data.properties.vod
+
+ if (vod) {
+ vod = vod.replace('v', '')
+ id += `_vod_${vod}`
+ }
+ }
+ }
  }
 
  return id
@@ -241,35 +263,181 @@ const getMediaData = (xhr, type) => {
  return result
  }
 
+ const parsedUrl = urlParse(xhr)
+ const query = parsedUrl && parsedUrl.query
+
+ if (!parsedUrl || !query) {
+ return null
+ }
+
  switch (type) {
  case ledgerMediaProviders.YOUTUBE:
  {
- const parsedUrl = urlParse(xhr)
- let query = null
+ result = queryString.parse(parsedUrl.query)
+ break
+ }
+ case ledgerMediaProviders.TWITCH:
+ {
+ result = queryString.parse(parsedUrl.query)
+ if (!result.data) {
+ result = null
+ break
+ }
+
+ let obj = Buffer.from(result.data, 'base64').toString('utf8')
+ if (obj == null) {
+ result = null
+ break
+ }
 
- if (parsedUrl && parsedUrl.query) {
- query = queryString.parse(parsedUrl.query)
+ try {
+ result = JSON.parse(obj)
+ } catch (error) {
+ result = null
+ console.error(error.toString())
  }
- result = query
  break
  }
  }
 
  return result
 }
 
-const getMediaDuration = (data, type) => {
+const getMediaDuration = (state, data, mediaKey, type) => {
  let duration = 0
+
+ if (data == null) {
+ return duration
+ }
+
  switch (type) {
- case ledgerMediaProviders.YOUTUBE: {
- duration = getYouTubeDuration(data)
- break
- }
+ case ledgerMediaProviders.YOUTUBE:
+ {
+ duration = getYouTubeDuration(data)
+ break
+ }
+ case ledgerMediaProviders.TWITCH:
+ {
+ duration = getTwitchDuration(state, data, mediaKey)
+ break
+ }
  }
 
  return duration
 }
 
+const generateMediaCacheData = (parsed, type) => {
+ let data = Immutable.Map()
+
+ if (parsed == null) {
+ return data
+ }
+
+ switch (type) {
+ case ledgerMediaProviders.TWITCH:
+ {
+ data = generateTwitchCacheData(parsed)
+ break
+ }
+ }
+
+ return data
+}
+
+const generateTwitchCacheData = (parsed) => {
+ if (parsed == null) {
+ return Immutable.Map()
+ }
+
+ if (parsed.properties) {
+ return Immutable.fromJS({
+ event: parsed.event,
+ time: parsed.properties.time
+ })
+ }
+
+ return Immutable.fromJS({
+ event: parsed.event
+ })
+}
+
+const getMediaFavicon = (providerName) => {
+ let image = null
+
+ if (!providerName) {
+ return image
+ }
+
+ providerName = providerName.toLowerCase()
+
+ switch (providerName) {
+ case ledgerMediaProviders.YOUTUBE:
+ {
+ image = require('../../../img/mediaProviders/youtube.png')
+ break
+ }
+ case ledgerMediaProviders.TWITCH:
+ {
+ image = require('../../../img/mediaProviders/twitch.svg')
+ break
+ }
+ }
+
+ return image
+}
+
+const getTwitchDuration = (state, data, mediaKey) => {
+ if (data == null || mediaKey == null) {
+ return 0
+ }
+
+ const previousData = ledgerVideoCache.getDataByVideoId(state, mediaKey)
+
+ if (previousData.isEmpty() && data.event === 'video-play') {
+ return milliseconds.second * 10
+ }
+
+ if (!data.properties) {
+ return 0
+ }
+
+ const oldMedia = ledgerVideoCache.getDataByVideoId(state, mediaKey)
+ let time = 0
+ const currentTime = parseFloat(data.properties.time)
+ const oldTime = parseFloat(previousData.get('time'))
+
+ if (
+ data.event === 'player_click_playpause' &&
+ oldMedia.get('event') !== 'player_click_playpause'
+ ) {
+ // User paused a video
+ time = currentTime - oldTime
+ } else if (previousData.get('event') === 'video-play') {
+ // From video play event to x event
+ time = currentTime - oldTime - 10
+ } else if (data.event === 'minute-watched') {
+ // Minute watched event
+ time = currentTime - oldTime
+ }
+
+ if (isNaN(time)) {
+ return 0
+ }
+
+ if (time < 0) {
+ return 0
+ }
+
+ if (time > 120) {
+ time = 120 // 2 minutes
+ }
+
+ // we get seconds back, so we need to convert it into ms
+ time = time * 1000
+
+ return Math.floor(time)
+}
+
 const getYouTubeDuration = (data) => {
  let time = 0
 
@@ -294,7 +462,7 @@ const getYouTubeDuration = (data) => {
  return parseInt(time)
 }
 
-const getMediaProvider = (url) => {
+const getMediaProvider = (url, firstPartyUrl, referrer) => {
  let provider = null
 
  if (url == null) {
@@ -303,7 +471,18 @@ const getMediaProvider = (url) => {
 
  // Youtube
  if (url.startsWith('https://www.youtube.com/api/stats/watchtime?')) {
- provider = ledgerMediaProviders.YOUTUBE
+ return ledgerMediaProviders.YOUTUBE
+ }
+
+ // Twitch
+ if (
+ (
+ (firstPartyUrl && firstPartyUrl.startsWith('https://www.twitch.tv')) ||
+ (referrer && referrer.startsWith('https://player.twitch.tv'))
+ ) &&
+ url.startsWith('https://api.mixpanel.com')
+ ) {
+ return ledgerMediaProviders.TWITCH
  }
 
  return provider
@@ -339,14 +518,18 @@ const getMethods = () => {
  getMediaData,
  getMediaKey,
  milliseconds,
- defaultMonthlyAmounts
+ defaultMonthlyAmounts,
+ getMediaFavicon,
+ generateMediaCacheData
  }
 
  let privateMethods = {}
 
  if (process.env.NODE_ENV === 'test') {
  privateMethods = {
- getYouTubeDuration
+ getYouTubeDuration,
+ getTwitchDuration,
+ generateTwitchCacheData
  }
  }