Merge pull request #13115 from brave/feature/publisher-request-api

Add publishers media info request API

app/browser/api/ledger.js

@@ -1408,6 +1408,77 @@ const clientprep = () => {
  _internal.verboseP = ledgerClient.prototype.boolion(process.env.LEDGER_PUBLISHER_VERBOSE)
 }
 
+/**
+ * This callback that we do it in roundtrips
+ *
+ * @callback roundtripCallback
+ * @param {Error|null} error - error when doing a request, null if no errors
+ * @param {object} response - response object that we get from request.request
+ * @param {any} payload - data that we get from the request
+ */
+
+/**
+ * Round trip for fetching data (scrap data from html) inside window process
+ * @param {object} params - contains params from roundtrip
+ * @param {string} params.url - url of the site that we want to scrap
+ * @param {string} params.verboseP - tells us if we want to log the process or not
+ * @param {roundtripCallback} callback - The callback that handles the response
+ */
+const roundTripFromWindow = (params, callback) => {
+ if (!callback) {
+ return
+ }
+
+ if (!params || !params.url) {
+ if (params && params.verboseP) {
+ console.log(`We are missing url. Params ${JSON.stringify(params)}`)
+ }
+
+ callback(new Error('Url is missing'))
+ return
+ }
+
+ request.fetchPublisherInfo(params.url, {
+ method: 'GET',
+ responseType: 'text',
+ headers: {
+ 'content-type': 'application/json; charset=utf-8'
+ }
+ }, (error, body) => {
+ if (error) {
+ if (params.verboseP) {
+ console.log(`roundTripFromWindow error: ${error.toString()}`)
+ }
+ return callback(error)
+ }
+
+ if (params.verboseP) {
+ console.log(`roundTripFromWindow success: ${JSON.stringify(body)}`)
+ }
+
+ return callback(null, null, body)
+ })
+}
+
+/**
+ * Round trip function for executing actions
+ * from the bat libraries (mostly server calls)
+ * @param {object} params - params that are directly tied to request.request
+ * @param {string} params.server - server url
+ * @param {string} params.method - HTTP method (GET, PUT, etc)
+ * @param {object} params.payload - payload that we want to send to the server
+ * @param {object} params.headers - HTTP headers
+ * @param {string} params.path - relative path to requested url
+ * @param {object} options
+ * @param {boolean} options.verboseP - tells us if we want to log the process or not
+ * @param {object} options.headers - headers that are used in the request.request
+ * @param {string} options.server - server url
+ * @param {boolean} options.binaryP - are we receiving binary payload back
+ * @param {boolean} options.rawP - are we receiving raw payload back
+ * @param {boolean} options.scrapeP - are we doping scraping
+ * @param {boolean} options.windowP - do we want to run this request in the window process
+ * @param {roundtripCallback} callback - The callback that handles the response
+ */
 const roundtrip = (params, options, callback) => {
  let parts = typeof params.server === 'string' ? urlParse(params.server)
  : typeof params.server !== 'undefined' ? params.server
@@ -1436,6 +1507,11 @@ const roundtrip = (params, options, callback) => {
  parts.pathname = parts.path
  }
 
+ if (options.windowP) {
+ roundTripFromWindow({url: urlFormat(parts), verboseP: options.verboseP}, callback)
+ return
+ }
+
  options = {
  url: urlFormat(parts),
  method: params.method,
@@ -2869,6 +2945,7 @@ const getMethods = () => {
  activityRoundTrip,
  pathName,
  onReferralInit,
+ roundTripFromWindow,
  onReferralCodeRead,
  onVerifiedPStatus
  }

app/extensions.js

@@ -38,7 +38,7 @@ let generateBraveManifest = () => {
  manifest_version: 2,
  version: '1.0',
  background: {
- scripts: [ 'content/scripts/idleHandler.js' ],
+ scripts: [ 'content/scripts/metaScraper.js', 'content/scripts/requestHandler.js', 'content/scripts/idleHandler.js' ],
  persistent: true
  },
  content_scripts: [
@@ -198,16 +198,19 @@ let generateBraveManifest = () => {
  'style-src': '\'self\' \'unsafe-inline\'',
  'font-src': '\'self\' data:',
  'img-src': '* data: file://*',
+ 'connect-src': 'https://www.youtube.com',
  'frame-src': '\'self\' https://brave.com'
  }
 
  if (process.env.NODE_ENV === 'development') {
  // allow access to webpack dev server resources
  let devServer = 'localhost:' + process.env.npm_package_config_port
  cspDirectives['default-src'] = '\'self\' http://' + devServer
- cspDirectives['connect-src'] = ['\'self\'',
+ cspDirectives['connect-src'] = cspDirectives['connect-src'] + [
+ ' \'self\'',
  'http://' + devServer,
- 'ws://' + devServer].join(' ')
+ 'ws://' + devServer
+ ].join(' ')
  cspDirectives['style-src'] = '\'self\' \'unsafe-inline\' http://' + devServer
  cspDirectives['font-src'] += ` http://${devServer}`
  }

app/extensions/brave/content/scripts/metaScraper.js

@@ -0,0 +1,102 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+ // Main repository
+ // Version 3.9.2
+// https://github.com/microlinkhq/metascraper
+
+// Image
+// https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-image
+
+// Author
+// https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-author
+
+// Title
+// https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-title
+
+// YouTube
+// https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-youtube
+
+const metaScraperRules = {
+ // Rules
+ getImageRules: () => {
+ const wrap = rule => ({htmlDom,url}) => {
+ const value = rule(htmlDom)
+ return requestHandlerApi.isUrl(value) && requestHandlerApi.getUrl(url, value)
+ }
+
+ return [
+ // Youtube
+ ({htmlDom,url}) => {
+ const {id,service} = requestHandlerApi.getVideoId(url)
+ return service === 'youtube' && id && requestHandlerApi.getThumbnailUrl(id)
+ },
+ // Regular
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="og:image:secure_url"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="og:image:url"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="og:image"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="twitter:image:src"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="twitter:image"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="sailthru.image.thumb"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="sailthru.image.full"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="sailthru.image"]'))),
+ wrap(html => requestHandlerApi.getValue(html.querySelectorAll('article img[src]'), requestHandlerApi.getSrc)),
+ wrap(html => requestHandlerApi.getValue(html.querySelectorAll('#content img[src]'), requestHandlerApi.getSrc)),
+ wrap(html => requestHandlerApi.getSrc(html.querySelector('img[alt*="author"]'))),
+ wrap(html => requestHandlerApi.getSrc(html.querySelector('img[src]')))
+ ]
+ },
+
+ getTitleRules: () => {
+ const wrap = rule => ({htmlDom}) => {
+ const value = rule(htmlDom)
+ return requestHandlerApi.isString(value) && requestHandlerApi.titleize(value)
+ }
+
+ return [
+ // Regular
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="og:title"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="twitter:title"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="sailthru.title"]'))),
+ wrap(html => requestHandlerApi.getText(html.querySelector('.post-title'))),
+ wrap(html => requestHandlerApi.getText(html.querySelector('.entry-title'))),
+ wrap(html => requestHandlerApi.getText(html.querySelector('[itemtype="http://schema.org/BlogPosting"] [itemprop="name"]'))),
+ wrap(html => requestHandlerApi.getText(html.querySelector('h1[class*="title"] a'))),
+ wrap(html => requestHandlerApi.getText(html.querySelector('h1[class*="title"]'))),
+ wrap(html => requestHandlerApi.getText(html.querySelector('title')))
+ ]
+ },
+
+ getAuthorRules: () => {
+ const wrap = rule => ({htmlDom}) => {
+ const value = rule(htmlDom)
+
+ return requestHandlerApi.isString(value) &&
+ !requestHandlerApi.isUrl(value, {relative: false}) &&
+ requestHandlerApi.titleize(value, {removeBy: true})
+ }
+
+ return [
+ // Youtube
+ wrap(html => requestHandlerApi.getText(html.querySelector('#owner-name'))),
+ wrap(html => requestHandlerApi.getText(html.querySelector('#channel-title'))),
+ wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[class*="user-info"]'))),
+ // Regular
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="author"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="article:author"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="author"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="sailthru.author"]'))),
+ wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[rel="author"]'))),
+ wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[itemprop*="author"] [itemprop="name"]'))),
+ wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[itemprop*="author"]'))),
+ wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="book:author"]'))),
+ requestHandlerApi.strict(wrap(html => requestHandlerApi.getValue(html.querySelectorAll('a[class*="author"]')))),
+ requestHandlerApi.strict(wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[class*="author"] a')))),
+ requestHandlerApi.strict(wrap(html => requestHandlerApi.getValue(html.querySelectorAll('a[href*="/author/"]')))),
+ wrap(html => requestHandlerApi.getValue(html.querySelectorAll('a[class*="screenname"]'))),
+ requestHandlerApi.strict(wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[class*="author"]')))),
+ requestHandlerApi.strict(wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[class*="byline"]'))))
+ ]
+ }
+}