Adds media publishers support

Auditors:

Test Plan:

app/browser/api/ledger.js

@@ -44,6 +44,7 @@ const request = require('../../../js/lib/request')
 const ledgerUtil = require('../../common/lib/ledgerUtil')
 const tabState = require('../../common/state/tabState')
 const pageDataUtil = require('../../common/lib/pageDataUtil')
+const ledgerVideoCahce = require('../../common/cache/ledgerVideoCache')
 
 // Caching
 let locationDefault = 'NOOP'
@@ -547,7 +548,8 @@ const getPublisherData = (result, scorekeeper) => {
  let data = {
  verified: result.options.verified || false,
  exclude: result.options.exclude || false,
- site: result.publisherKey,
+ publisherURL: result.publisherKey,
+ siteName: result.publisherKey,
  views: result.visits,
  duration: duration,
  daysSpent: 0,
@@ -563,6 +565,13 @@ const getPublisherData = (result, scorekeeper) => {
  // still generate publisherURL.
  data.publisherURL = (result.protocol || 'http:') + '//' + result.publisherKey
 
+ // media publisher
+ if (result.faviconName) {
+ const on = locale.translation('on').toLowerCase()
+ data.publisherURL = result.publisherURL
+ data.siteName = `${result.faviconName} ${on} ${result.providerName}`
+ }
+
  if (duration >= miliseconds.day) {
  data.daysSpent = Math.max(Math.round(duration / miliseconds.day), 1)
  } else if (duration >= miliseconds.hour) {
@@ -833,7 +842,7 @@ const excludeP = (publisherKey, callback) => {
  return done(err, result)
  }
 
- let props = ledgerPublisher.getPublisherProps('https://' + publisherKey)
+ let props = ledgerPublisher.getPublisherProps(publisherKey)
  if (!props) return done()
 
  v2RulesetDB.createReadStream({lt: 'domain:'}).on('data', (data) => {
@@ -874,30 +883,26 @@ const excludeP = (publisherKey, callback) => {
  })
 }
 
-const addVisit = (state, startTimestamp, location, tabId) => {
+const addSiteVisit = (state, timestamp, location, tabId) => {
  if (!synopsis) {
  return state
  }
 
  location = pageDataUtil.getInfoKey(location)
  const locationData = ledgerState.getLocation(state, location)
- const timestamp = new Date().getTime()
+ const duration = new Date().getTime() - timestamp
  if (_internal.verboseP) {
  console.log(
  `locations[${location}]=${JSON.stringify(locationData, null, 2)} ` +
- `duration=${(timestamp - startTimestamp)} msec tabId= ${tabId}`
+ `duration=${(duration)} msec tabId= ${tabId}`
  )
  }
- if (locationData.isEmpty() || !tabId) {
- return state
- }
 
- let publisherKey = locationData.get('publisher')
- if (!publisherKey) {
+ if (locationData.isEmpty()) {
  return state
  }
 
- let duration = timestamp - startTimestamp
+ let publisherKey = locationData.get('publisher')
  let revisitP = false
 
  if (duration >= getSetting(settings.PAYMENTS_MINIMUM_VISIT_TIME)) {
@@ -917,13 +922,19 @@ const addVisit = (state, startTimestamp, location, tabId) => {
  }
  }
 
+ return saveVisit(state, publisherKey, duration, revisitP)
+}
+
+const saveVisit = (state, publisherKey, duration, revisited) => {
+ if (!synopsis || !publisherKey) {
+ return state
+ }
+
  if (_internal.verboseP) {
- console.log('\nadd publisher ' + publisherKey + ': ' + (duration / 1000) + ' sec' + ' revisitP=' + revisitP + ' state=' +
- JSON.stringify(underscore.extend({location: location}, visitsByPublisher[publisherKey][location]),
- null, 2))
+ console.log('\nadd publisher ' + publisherKey + ': ' + (duration / 1000) + ' sec' + ' revisitP=' + revisited)
  }
 
- synopsis.addPublisher(publisherKey, {duration: duration, revisitP: revisitP})
+ synopsis.addPublisher(publisherKey, {duration: duration, revisitP: revisited})
  state = ledgerState.setPublisher(state, publisherKey, synopsis.publishers[publisherKey])
  state = updatePublisherInfo(state)
  state = verifiedP(state, publisherKey, (error, result) => {
@@ -957,7 +968,7 @@ const addNewLocation = (state, location, tabId = tabState.TAB_ID_NONE, keepInfo
 
  // Add visit to the ledger when we are not in a private tab
  if (!isPrivate && !tabFromState.isEmpty() && ledgerUtil.shouldTrackView(tabFromState)) {
- state = addVisit(state, currentTimestamp, currentUrl, currentTabId)
+ state = addSiteVisit(state, currentTimestamp, currentUrl, currentTabId)
  }
  }
 
@@ -1504,7 +1515,8 @@ const roundtrip = (params, options, callback) => {
  let parts = typeof params.server === 'string' ? urlParse(params.server)
  : typeof params.server !== 'undefined' ? params.server
  : typeof options.server === 'string' ? urlParse(options.server) : options.server
- const rawP = options.rawP
+ const binaryP = options.binaryP
+ const rawP = binaryP || options.rawP
 
  if (!params.method) params.method = 'GET'
  parts = underscore.extend(underscore.pick(parts, ['protocol', 'hostname', 'port']),
@@ -1531,7 +1543,7 @@ const roundtrip = (params, options, callback) => {
  url: urlFormat(parts),
  method: params.method,
  payload: params.payload,
- responseType: 'text',
+ responseType: binaryP ? 'binary' : 'text',
  headers: underscore.defaults(params.headers || {}, {'content-type': 'application/json; charset=utf-8'}),
  verboseP: options.verboseP
  }
@@ -2460,6 +2472,113 @@ const transitionWalletToBat = () => {
  }
 }
 
+let currentMediaKey = null
+const onMediaRequest = (state, xhr, type) => {
+ if (!xhr) {
+ return state
+ }
+
+ const parsed = ledgerUtil.getMediaData(xhr, type)
+ const mediaId = ledgerUtil.getMediaId(parsed, type)
+ const mediaKey = ledgerUtil.getMediaKey(mediaId, type)
+ const duration = ledgerUtil.getMediaDuration(parsed, type)
+
+ if (mediaId == null || duration == null || mediaKey == null) {
+ return state
+ }
+
+ if (!ledgerPublisher) {
+ ledgerPublisher = require('bat-publisher')
+ }
+
+ let revisited = true
+ if (mediaKey !== currentMediaKey) {
+ revisited = false
+ currentMediaKey = mediaKey
+ }
+
+ const cache = ledgerVideoCahce.getDataByVideoId(state, mediaKey)
+
+ if (!cache.isEmpty()) {
+ return saveVisit(state, cache.get('publisher'), duration, revisited)
+ }
+
+ const options = underscore.extend({roundtrip: roundtrip}, clientOptions)
+ const mediaProps = {
+ mediaId,
+ providerName: type
+ }
+
+ ledgerPublisher.getMedia.getPublisherFromMediaProps(mediaProps, options, (error, response) => {
+ if (error) {
+ console.error('Error while getting publisher from media', error.toString())
+ return
+ }
+
+ // publisher not found
+ if (!response) {
+ return
+ }
+
+ appActions.onLedgerMediaPublisher(mediaKey, response, duration, revisited)
+ })
+
+ return state
+}
+
+const onMediaPublisher = (state, mediaKey, response, duration, revisited) => {
+ const publisherKey = response.get('publisher')
+ if (publisherKey == null) {
+ return state
+ }
+
+ const minDuration = getSetting(settings.PAYMENTS_MINIMUM_VISIT_TIME)
+ duration = parseInt(duration)
+ if (duration > 0 && duration < minDuration) {
+ duration = minDuration
+ }
+
+ let publisher = ledgerState.getPublisher(state, publisherKey)
+ const faviconName = response.get('faviconName')
+ const faviconURL = response.get('faviconURL')
+ const publisherURL = response.get('publisherURL')
+
+ if (publisher.isEmpty()) {
+ synopsis.initPublisher(publisherKey)
+
+ synopsis.publishers[publisherKey].faviconName = faviconName
+ synopsis.publishers[publisherKey].faviconURL = faviconURL
+ synopsis.publishers[publisherKey].publisherURL = publisherURL
+ synopsis.publishers[publisherKey].providerName = response.get('providerName')
+
+ if (synopsis.publishers[publisherKey]) {
+ state = ledgerState.setPublisher(state, publisherKey, synopsis.publishers[publisherKey])
+ }
+
+ if (!getSetting(settings.PAYMENTS_SITES_AUTO_SUGGEST)) {
+ appActions.onPublisherOptionUpdate(publisherKey, 'exclude', true)
+ savePublisherOption(publisherKey, 'exclude', true)
+ } else {
+ excludeP(publisherKey, (unused, exclude) => {
+ appActions.onPublisherOptionUpdate(publisherKey, 'exclude', exclude)
+ savePublisherOption(publisherKey, 'exclude', exclude)
+ })
+ }
+ } else {
+ synopsis.publishers[publisherKey].faviconName = faviconName
+ synopsis.publishers[publisherKey].faviconURL = faviconURL
+ state = ledgerState.setPublishersProp(state, publisherKey, 'faviconName', faviconName)
+ state = ledgerState.setPublishersProp(state, publisherKey, 'faviconURL', faviconURL)
+ }
+
+ // Add to cache
+ state = ledgerVideoCahce.setCacheByVideoId(state, mediaKey, response)
+
+ state = saveVisit(state, publisherKey, duration, revisited)
+
+ return state
+}
+
 const getMethods = () => {
  const publicMethods = {
  backupKeys,
@@ -2493,15 +2612,17 @@ const getMethods = () => {
  deleteSynopsis,
  transitionWalletToBat,
  getNewClient,
- savePublisherData
+ savePublisherData,
+ onMediaRequest,
+ onMediaPublisher
  }
 
  let privateMethods = {}
 
  if (process.env.NODE_ENV === 'test') {
  privateMethods = {
  enable,
- addVisit,
+ addSiteVisit,
  clearVisitsByPublisher: function () {
  visitsByPublisher = {}
  },

app/browser/reducers/ledgerReducer.js

@@ -405,6 +405,22 @@ const ledgerReducer = (state, action, immutableAction) => {
  }
  break
  }
+ case appConstants.APP_ON_LEDGER_MEDIA_DATA:
+ {
+ state = ledgerApi.onMediaRequest(state, action.get('url'), action.get('type'))
+ break
+ }
+ case appConstants.APP_ON_LEDGER_MEDIA_PUBLISHER:
+ {
+ state = ledgerApi.onMediaPublisher(
+ state,
+ action.get('mediaKey'),
+ action.get('response'),
+ action.get('duration'),
+ action.get('revisited')
+ )
+ break
+ }
  }
  return state
 }

app/common/cache/ledgerVideoCache.js

@@ -0,0 +1,25 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const Immutable = require('immutable')
+
+const getDataByVideoId = (state, key) => {
+ if (key == null) {
+ return Immutable.Map()
+ }
+
+ return state.getIn(['cache', 'ledgerVideos', key]) || Immutable.Map()
+}
+
+const setCacheByVideoId = (state, key, data) => {
+ if (key == null) {
+ return state
+ }
+
+ return state.setIn(['cache', 'ledgerVideos', key], data)
+}
+
+module.exports = {
+ getDataByVideoId,
+ setCacheByVideoId
+}

app/common/constants/ledgerMediaProviders.js

@@ -0,0 +1,12 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const mapValuesByKeys = require('../../../js/lib/functional').mapValuesByKeys
+
+const _ = null
+const providers = {
+ YOUTUBE: _
+}
+
+module.exports = mapValuesByKeys(providers)