Skip to content
This repository has been archived by the owner on May 9, 2022. It is now read-only.

add twitch#channel URI #11

Merged
merged 2 commits into from
Feb 13, 2018
Merged

add twitch#channel URI #11

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dfd0f40
add twitch#channel URI
mrose17 Feb 9, 2018
8dc0b6d
Remove inadvertent debug statement
mrose17 Feb 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 52 additions & 9 deletions getMedia.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,14 @@ const getPublisherFromMediaProps = (mediaProps, options, callback) => {
}

const mappers = {
twitch: (mediaProps) => {
const mediaId = mediaProps.mediaId

if (!mediaId) throw new Error('expecting mediaId for provider Twitch')

return ('https://www.twitch.tv/videos/' + mediaId)
},

youtube: (mediaProps) => {
const mediaId = mediaProps.mediaId

Expand All @@ -37,7 +45,7 @@ const mappers = {
}

const getPublisherFromMediaURL = (mediaURL, options, callback) => {
let parts, providers
let domains, hostname, parts, providers

if (typeof options === 'function') {
callback = options
Expand All @@ -51,12 +59,17 @@ const getPublisherFromMediaURL = (mediaURL, options, callback) => {
else throw new Error('security audit requires options.roundtrip for non-debug use')

parts = url.parse(mediaURL)
if ((parts) && (parts.protocol !== 'https:')) return setTimeout(() => { callback(new Error('non-https URL'), null) }, 0)
if ((!parts) || (parts.protocol !== 'https:')) return setTimeout(() => { callback(new Error('invalid URL'), null) }, 0)

hostname = parts.hostname
domains = [ hostname, tldjs.getDomain(hostname) ]

if (hostname.indexOf('www.') === 0) domains.push('api.' + hostname.substr(4))

providers = underscore.filter(options.ruleset, (rule) => {
const schemes = rule.schemes

if (!schemes.length) return ((parts) && (tldjs.getDomain(parts.hostname) === rule.domain))
if (!schemes.length) return (domains.indexOf(rule.domain) !== -1)

for (let scheme in schemes) if (mediaURL.match(new RegExp(scheme.replace(/\*/g, '(.*)'), 'i'))) return true
})
Expand All @@ -78,6 +91,7 @@ const getPublisherFromProviders = (providers, mediaURL, options, firstErr, callb
if (!resolver) return done(new Error('no resolver for ' + provider.provider_name))

parts = url.parse(provider.url + '?' + querystring.stringify({ format: 'json', url: mediaURL }))

retryTrip({
server: parts.protocol + '//' + parts.host,
path: parts.path,
Expand All @@ -91,13 +105,13 @@ const getPublisherFromProviders = (providers, mediaURL, options, firstErr, callb
}

const resolvers = {
YouTube: (providers, mediaURL, options, payload, firstErr, callback) => {
_channel: (providers, mediaURL, options, payload, firstErr, callback) => {
const provider = underscore.first(providers)
const parts = url.parse(payload.author_url)
let paths

paths = parts && parts.pathname.split('/')
if ((!paths) || (paths.length !== 3)) throw new Error('invalid author_url: ' + payload.author_url)
if ((!paths) || (!payload._channel.validP(paths))) throw new Error('invalid author_url: ' + payload.author_url)

cachedTrip({
server: parts.protocol + '//' + parts.host,
Expand All @@ -107,12 +121,12 @@ const resolvers = {
if (err) return next(providers, mediaURL, options, firstErr || err, callback)

metascraper.scrapeHtml(body).then((result) => {
console.log('result: ' + JSON.stringify(result, null, 2))
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this contains potentially sensitive information about the page and should not be logged by default

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oops! debug thing not removed. fixed.


const parts = url.parse(result.url)
const cpaths = parts && parts.pathname.split('/')
const channel = (parts.pathname === parts.path) && (cpaths.length === 3) && (cpaths[1] === 'channel')
? cpaths[2] : paths[2]
const channel = payload._channel.get(paths, parts)
const publisherInfo = {
publisher: 'youtube#channel:' + channel,
publisher: payload._channel.providerName + '#channel:' + channel,
publisherType: 'provider',
publisherURL: payload.author_url + '/videos',
providerName: provider.provider_name,
Expand Down Expand Up @@ -154,6 +168,35 @@ const resolvers = {
next(providers, mediaURL, options, firstErr || err, callback)
})
})
},

Twitch: (providers, mediaURL, options, payload, firstErr, callback) => {
resolvers._channel(providers, mediaURL, options, underscore.extend({
_channel: {
providerName: 'twitch',
param1: 2,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are we using this one?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes. that's the point of the PR. i refactored so the twitch and youtube code could maximally reuse code.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

where do we use it? in other library? I can't find it in this PR

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

take a look at media/provider.json!

validP: (paths) => { return (paths.length === 2) },
get: (paths, parts) => {
const cpaths = parts && parts.pathname.split('/')

return ((parts.pathname === parts.path) && (cpaths.length === 2) ? cpaths[1] : paths[1])
}
}
}, payload), firstErr, callback)
},

YouTube: (providers, mediaURL, options, payload, firstErr, callback) => {
resolvers._channel(providers, mediaURL, options, underscore.extend({
_channel: {
providerName: 'youtube',
validP: (paths) => { return (paths.length === 3) },
get: (paths, parts) => {
const cpaths = parts && parts.pathname.split('/')

return ((parts.pathname === parts.path) && (cpaths.length === 3) && (cpaths[1] === 'channel') ? cpaths[2] : paths[2])
}
}
}, payload), firstErr, callback)
}
}

Expand Down
21 changes: 17 additions & 4 deletions index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,19 @@ const getPublisher = (location, markup, ruleset) => {
}
}

const publisherURLs = {
twitch: (props) => {
if (props.providerSuffix === 'channel') return ('https://www.twitch.tv/' + props.providerValue)
},

youtube: (props) => {
if (props.providerSuffix === 'channel') return ('https://www.youtube.com/channel/' + props.providerValue)
}
}

const getPublisherProps = (publisher) => {
const provider = providerRE.exec(publisher)
let props
let f, props, providerURL

if (provider) {
props = {
Expand All @@ -89,15 +99,18 @@ const getPublisherProps = (publisher) => {
providerSuffix: provider[2],
providerValue: querystring.unescape(provider[3])
}

f = publisherURLs[props.providerName.toLowerCase()]
providerURL = f && f(props)
if (providerURL) props.URL = providerURL

underscore.extend(props, {
TLD: props.publisher.split(':')[0],
SLD: props.publisher,
RLD: props.providerValue,
QLD: ''
})
if ((props.providerName.toLowerCase() === 'youtube') && (props.providerSuffix === 'channel')) {
props.URL = 'https://www.youtube.com/channel/' + props.providerValue
}

return props
}

Expand Down
4 changes: 3 additions & 1 deletion make-providers.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@ const underscore = require('underscore')

const rules = require('./node_modules/oembed-parser/src/utils/providers.json')

const providers = [ 'Twitch', 'YouTube' ]

const ruleset = []
rules.forEach((rule) => {
let domain, endpoint, match

if (rule.provider_name !== 'YouTube') return
if (providers.indexOf(rule.provider_name) === -1) return

endpoint = rule.endpoints[0].url
match = endpoint.match(/:\/\/(www[0-9]?\.)?(.[^/:]+)/i)
Expand Down
7 changes: 7 additions & 0 deletions media/providers.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
[
{
"provider_name": "Twitch",
"provider_url": "https://www.twitch.tv",
"schemes": [],
"domain": "api.twitch.tv",
"url": "https://api.twitch.tv/v4/oembed"
},
{
"provider_name": "YouTube",
"provider_url": "https://www.youtube.com/",
Expand Down
3 changes: 2 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "bat-publisher",
"version": "2.0.3",
"version": "2.0.4",
"description": "Routines to identify publishers for the BAT.",
"main": "index.js",
"scripts": {
Expand Down Expand Up @@ -47,6 +47,7 @@
"level": "1.7.0",
"npm-check-updates": "^2.12.1",
"nsp": "^2.8.0",
"oembed-parser": "^1.0.8",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we need security review for this one. I see that there is no new usage, where we just missing it from the dependencies?

cc @diracdeltas

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/brave/internal/issues/203

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should not use ombed-parser in general since it uses Node's networking stack (https://github.com/ndaidong/oembed-parser/blob/master/src/utils/fetchEmbed.js), but this usage is ok for now since it only uses the provider.json rules file

"standard": "10.0.3",
"tap": "^10.7.2"
},
Expand Down