From 52af27b0aa4e3b3582d52d4894fd269673888708 Mon Sep 17 00:00:00 2001 From: MarcoCoreDuo <90222533+MarcoCoreDuo@users.noreply.github.com> Date: Fri, 17 May 2024 17:31:53 +0200 Subject: [PATCH] Switch ytdl-core to play-dl --- package.json | 3 +- patches/ytdl-core+4.11.5.patch | 172 --------------------------------- src/services/player.ts | 62 ++++++------ yarn.lock | 73 +++++++++----- 4 files changed, 84 insertions(+), 226 deletions(-) delete mode 100644 patches/ytdl-core+4.11.5.patch diff --git a/package.json b/package.json index 6a80e0abc..c43165012 100644 --- a/package.json +++ b/package.json @@ -108,6 +108,7 @@ "libsodium-wrappers": "^0.7.9", "make-dir": "^3.1.0", "node-emoji": "^1.10.0", + "node-fetch": "^3.3.2", "nodesplash": "^0.1.1", "ora": "^6.1.0", "p-event": "^5.0.1", @@ -117,6 +118,7 @@ "pagination.djs": "^4.0.10", "parse-duration": "1.0.2", "patch-package": "^8.0.0", + "play-dl": "^1.9.7", "postinstall-postinstall": "^2.1.0", "read-pkg": "7.1.0", "reflect-metadata": "^0.1.13", @@ -126,7 +128,6 @@ "sync-fetch": "^0.3.1", "tsx": "3.8.2", "xbytes": "^1.7.0", - "ytdl-core": "^4.11.5", "ytsr": "^3.8.4" }, "resolutions": { diff --git a/patches/ytdl-core+4.11.5.patch b/patches/ytdl-core+4.11.5.patch deleted file mode 100644 index 7d3abf378..000000000 --- a/patches/ytdl-core+4.11.5.patch +++ /dev/null @@ -1,172 +0,0 @@ -diff --git a/node_modules/ytdl-core/lib/sig.js b/node_modules/ytdl-core/lib/sig.js -index eb7bfaa..b2eee87 100644 ---- a/node_modules/ytdl-core/lib/sig.js -+++ b/node_modules/ytdl-core/lib/sig.js -@@ -3,6 +3,9 @@ const Cache = require('./cache'); - const utils = require('./utils'); - const vm = require('vm'); - -+ -+let nTransformWarning = false; -+ - // A shared cache to keep track of html5player js functions. - exports.cache = new Cache(); - -@@ -23,6 +26,49 @@ exports.getFunctions = (html5playerfile, options) => exports.cache.getOrSet(html - return functions; - }); - -+// eslint-disable-next-line max-len -+// https://github.com/TeamNewPipe/NewPipeExtractor/blob/41c8dce452aad278420715c00810b1fed0109adf/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java#L816 -+const DECIPHER_REGEXPS = [ -+ '(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2,})\\s*=\\s*function\\(\\s*a\\s*\\)' + -+ '\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*""\\s*\\)', -+ '\\bm=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(h\\.s\\)\\)', -+ '\\bc&&\\(c=([a-zA-Z0-9$]{2,})\\(decodeURIComponent\\(c\\)\\)', -+ '([\\w$]+)\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(""\\)\\s*;', -+ '\\b([\\w$]{2,})\\s*=\\s*function\\((\\w+)\\)\\{\\s*\\2=\\s*\\2\\.split\\(""\\)\\s*;', -+ '\\bc\\s*&&\\s*d\\.set\\([^,]+\\s*,\\s*(:encodeURIComponent\\s*\\()([a-zA-Z0-9$]+)\\(', -+]; -+ -+const DECIPHER_ARGUMENT = 'sig'; -+const N_ARGUMENT = 'ncode'; -+ -+const matchGroup1 = (regex, str) => { -+ const match = str.match(new RegExp(regex)); -+ if (!match) throw new Error(`Could not match ${regex}`); -+ return match[1]; -+}; -+ -+const getFuncName = (body, regexps) => { -+ try { -+ let fn; -+ for (const regex of regexps) { -+ try { -+ fn = matchGroup1(regex, body); -+ const idx = fn.indexOf('[0]'); -+ if (idx > -1) fn = matchGroup1(`${fn.slice(0, 3)}=\\[([a-zA-Z0-9$\\[\\]]{2,})\\]`, body); -+ } catch (err) { -+ continue; -+ } -+ } -+ if (!fn || fn.includes('[')) throw Error("Couldn't find fn name"); -+ return fn; -+ } catch (e) { -+ throw Error(`Please open an issue on ytdl-core GitHub: ${e.message}`); -+ } -+}; -+ -+const getDecipherFuncName = body => getFuncName(body, DECIPHER_REGEXPS); -+ -+ - /** - * Extracts the actions that should be taken to decipher a signature - * and tranform the n parameter -@@ -31,44 +77,45 @@ exports.getFunctions = (html5playerfile, options) => exports.cache.getOrSet(html - * @returns {Array.} - */ - exports.extractFunctions = body => { -+ body = body.replace(/\n|\r/g, ''); - const functions = []; -- const extractManipulations = caller => { -- const functionName = utils.between(caller, `a=a.split("");`, `.`); -- if (!functionName) return ''; -- const functionStart = `var ${functionName}={`; -- const ndx = body.indexOf(functionStart); -- if (ndx < 0) return ''; -- const subBody = body.slice(ndx + functionStart.length - 1); -- return `var ${functionName}=${utils.cutAfterJS(subBody)}`; -- }; -+ // This is required function, so we can't continue if it's not found. - const extractDecipher = () => { -- const functionName = utils.between(body, `a.set("alr","yes");c&&(c=`, `(decodeURIC`); -- if (functionName && functionName.length) { -- const functionStart = `${functionName}=function(a)`; -- const ndx = body.indexOf(functionStart); -- if (ndx >= 0) { -- const subBody = body.slice(ndx + functionStart.length); -- let functionBody = `var ${functionStart}${utils.cutAfterJS(subBody)}`; -- functionBody = `${extractManipulations(functionBody)};${functionBody};${functionName}(sig);`; -- functions.push(functionBody); -- } -+ const decipherFuncName = getDecipherFuncName(body); -+ try { -+ const functionPattern = `(${decipherFuncName.replace(/\$/g, '\\$')}=function\\([a-zA-Z0-9_]+\\)\\{.+?\\})`; -+ const decipherFunction = `var ${matchGroup1(functionPattern, body)};`; -+ const helperObjectName = matchGroup1(';([A-Za-z0-9_\\$]{2,})\\.\\w+\\(', decipherFunction) -+ .replace(/\$/g, '\\$'); -+ const helperPattern = `(var ${helperObjectName}=\\{[\\s\\S]+?\\}\\};)`; -+ const helperObject = matchGroup1(helperPattern, body); -+ const callerFunction = `${decipherFuncName}(${DECIPHER_ARGUMENT});`; -+ const resultFunction = helperObject + decipherFunction + callerFunction; -+ functions.push(resultFunction); -+ } catch (err) { -+ throw Error(`Could not parse decipher function: ${err}`); - } - }; -- const extractNCode = () => { -- let functionName = utils.between(body, `&&(b=a.get("n"))&&(b=`, `(b)`); -- if (functionName.includes('[')) functionName = utils.between(body, `var ${functionName.split('[')[0]}=[`, `]`); -- if (functionName && functionName.length) { -- const functionStart = `${functionName}=function(a)`; -- const ndx = body.indexOf(functionStart); -- if (ndx >= 0) { -- const subBody = body.slice(ndx + functionStart.length); -- const functionBody = `var ${functionStart}${utils.cutAfterJS(subBody)};${functionName}(ncode);`; -- functions.push(functionBody); -+ // This is optional, so we can continue if it's not found, but it will bottleneck the download. -+ const extractNTransform = () => { -+ let nFuncName = utils.between(body, `(b=a.get("n"))&&(b=`, `(b)`); -+ if (nFuncName.includes('[')) nFuncName = utils.between(body, `${nFuncName.split('[')[0]}=[`, `]`); -+ if (nFuncName && nFuncName.length) { -+ const nBegin = `${nFuncName}=function(a)`; -+ const nEnd = '.join("")};'; -+ const nFunction = utils.between(body, nBegin, nEnd); -+ if (nFunction) { -+ const callerFunction = `${nFuncName}(${N_ARGUMENT});`; -+ const resultFunction = nBegin + nFunction + nEnd + callerFunction; -+ functions.push(resultFunction); -+ } else if (!nTransformWarning) { -+ console.warn('Could not parse n transform function, please report it on @distube/ytdl-core GitHub.'); -+ nTransformWarning = true; - } - } - }; - extractDecipher(); -- extractNCode(); -+ extractNTransform(); - return functions; - }; - -@@ -82,22 +129,25 @@ exports.extractFunctions = body => { - exports.setDownloadURL = (format, decipherScript, nTransformScript) => { - const decipher = url => { - const args = querystring.parse(url); -- if (!args.s || !decipherScript) return args.url; -+ if (!args.s) return args.url; - const components = new URL(decodeURIComponent(args.url)); -- components.searchParams.set(args.sp ? args.sp : 'signature', -- decipherScript.runInNewContext({ sig: decodeURIComponent(args.s) })); -+ const context = {}; -+ context[DECIPHER_ARGUMENT] = decodeURIComponent(args.s); -+ components.searchParams.set(args.sp || 'sig', decipherScript.runInNewContext(context)); - return components.toString(); - }; -- const ncode = url => { -+ const nTransform = url => { - const components = new URL(decodeURIComponent(url)); - const n = components.searchParams.get('n'); - if (!n || !nTransformScript) return url; -- components.searchParams.set('n', nTransformScript.runInNewContext({ ncode: n })); -+ const context = {}; -+ context[N_ARGUMENT] = n; -+ components.searchParams.set('n', nTransformScript.runInNewContext(context)); - return components.toString(); - }; - const cipher = !format.url; - const url = format.url || format.signatureCipher || format.cipher; -- format.url = cipher ? ncode(decipher(url)) : ncode(url); -+ format.url = cipher ? nTransform(decipher(url)) : nTransform(url); - delete format.signatureCipher; - delete format.cipher; - }; diff --git a/src/services/player.ts b/src/services/player.ts index 2ec177c74..f77e7272c 100644 --- a/src/services/player.ts +++ b/src/services/player.ts @@ -1,10 +1,11 @@ import {VoiceChannel, Snowflake} from 'discord.js'; import {Readable} from 'stream'; import hasha from 'hasha'; -import ytdl, {videoFormat} from 'ytdl-core'; +import {InfoData, video_basic_info} from 'play-dl'; import {WriteStream} from 'fs-capacitor'; import ffmpeg from 'fluent-ffmpeg'; import shuffle from 'array-shuffle'; +import fetch from 'node-fetch'; import { AudioPlayer, AudioPlayerState, @@ -57,8 +58,6 @@ export interface PlayerEvents { statusChange: (oldStatus: STATUS, newStatus: STATUS) => void; } -type YTDLVideoFormat = videoFormat & {loudnessDb?: number}; - export const DEFAULT_VOLUME = 100; export default class { @@ -437,55 +436,56 @@ export default class { const ffmpegInputOptions: string[] = []; let shouldCacheVideo = false; - let format: YTDLVideoFormat | undefined; + let format: InfoData['format'][0] | undefined; ffmpegInput = await this.fileCache.getPathFor(this.getHashForCache(song.url)); if (!ffmpegInput) { // Not yet cached, must download - const info = await ytdl.getInfo(song.url); + const info = await video_basic_info(song.url); - const formats = info.formats as YTDLVideoFormat[]; + if (info.LiveStreamData.isLive) { + const hlsUrl = info.LiveStreamData.hlsManifestUrl; - const filter = (format: ytdl.videoFormat): boolean => format.codecs === 'opus' && format.container === 'webm' && format.audioSampleRate !== undefined && parseInt(format.audioSampleRate, 10) === 48000; + if (hlsUrl === null) { + throw new Error('No HLS manifest URL found.'); + } - format = formats.find(filter); + const audioBitrates: Record = {128: 96, 127: 96, 120: 128, 96: 256, 95: 256, 94: 128, 93: 128}; - const nextBestFormat = (formats: ytdl.videoFormat[]): ytdl.videoFormat | undefined => { - if (formats[0].isLive) { - formats = formats.sort((a, b) => (b as unknown as {audioBitrate: number}).audioBitrate - (a as unknown as {audioBitrate: number}).audioBitrate); // Bad typings + let formats: Array<{itag: number; url: string; audioBitrate?: number; loudnessDb: undefined}> = []; - return formats.find(format => [128, 127, 120, 96, 95, 94, 93].includes(parseInt(format.itag as unknown as string, 10))); // Bad typings - } + const m3u8_data = await fetch(hlsUrl).then(async res => res.text()); - formats = formats - .filter(format => format.averageBitrate) - .sort((a, b) => { - if (a && b) { - return b.averageBitrate! - a.averageBitrate!; + m3u8_data + .split('\n') + .filter(line => /^https?:\/\//.test(line)) + .forEach(line => { + let itag: RegExpExecArray | number | null = /\/itag\/(\d+)\//.exec(line); + if (itag !== null) { + itag = parseInt(itag[1], 10); + formats.unshift({itag, url: line, audioBitrate: audioBitrates[itag], loudnessDb: undefined}); } - - return 0; }); - return formats.find(format => !format.bitrate) ?? formats[0]; - }; - if (!format) { - format = nextBestFormat(info.formats); + formats = formats.sort((a, b) => (b as unknown as {audioBitrate: number}).audioBitrate - (a as unknown as {audioBitrate: number}).audioBitrate); - if (!format) { - // If still no format is found, throw - throw new Error('Can\'t find suitable format.'); - } + format = formats.find(format => [128, 127, 120, 96, 95, 94, 93].includes(format.itag)); + } else { + format = info.format.at(info.format.length - 1); } - debug('Using format', format); + if (!format) { + // If no format is found, throw + throw new Error('Can\'t find suitable format.'); + } - ffmpegInput = format.url; + debug('Using format', format); + ffmpegInput = format.url!; // Don't cache livestreams or long videos const MAX_CACHE_LENGTH_SECONDS = 30 * 60; // 30 minutes - shouldCacheVideo = !info.player_response.videoDetails.isLiveContent && parseInt(info.videoDetails.lengthSeconds, 10) < MAX_CACHE_LENGTH_SECONDS && !options.seek; + shouldCacheVideo = !info.video_details.live && info.video_details.durationInSec < MAX_CACHE_LENGTH_SECONDS && !options.seek; debug(shouldCacheVideo ? 'Caching video' : 'Not caching video'); diff --git a/yarn.lock b/yarn.lock index 2cebf9011..70613b710 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1249,6 +1249,11 @@ data-uri-to-buffer@3: resolved "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz" integrity sha512-WboRycPNsVw3B3TL559F7kuBUM4d8CgMEvk6xEJlOp7OBPjt6G7z8WMWlD2rOFZLk6OYfFIUGsCOWzcQH9K2og== +data-uri-to-buffer@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz#d8feb2b2881e6a4f58c2e08acfd0e2834e26222e" + integrity sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A== + data-view-buffer@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/data-view-buffer/-/data-view-buffer-1.0.1.tgz#8ea6326efec17a2e42620696e671d7d5a8bc66b2" @@ -2072,6 +2077,14 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" +fetch-blob@^3.1.2, fetch-blob@^3.1.4: + version "3.2.0" + resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.2.0.tgz#f09b8d4bbd45adc6f0c20b7e787e793e309dcce9" + integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ== + dependencies: + node-domexception "^1.0.0" + web-streams-polyfill "^3.0.3" + figures@^3.0.0: version "3.2.0" resolved "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz" @@ -2199,6 +2212,13 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +formdata-polyfill@^4.0.10: + version "4.0.10" + resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423" + integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g== + dependencies: + fetch-blob "^3.1.2" + formidable@^1.2.2: version "1.2.6" resolved "https://registry.npmjs.org/formidable/-/formidable-1.2.6.tgz" @@ -3308,14 +3328,6 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -m3u8stream@^0.8.6: - version "0.8.6" - resolved "https://registry.npmjs.org/m3u8stream/-/m3u8stream-0.8.6.tgz" - integrity sha512-LZj8kIVf9KCphiHmH7sbFQTVe4tOemb202fWwvJwR9W5ENW/1hxJN6ksAWGhQgSBSa3jyWhnjKU1Fw1GaOdbyA== - dependencies: - miniget "^4.2.2" - sax "^1.2.4" - macos-release@^2.5.0: version "2.5.0" resolved "https://registry.npmjs.org/macos-release/-/macos-release-2.5.0.tgz" @@ -3462,6 +3474,11 @@ node-addon-api@^5.0.0: resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.0.0.tgz" integrity sha512-CvkDw2OEnme7ybCykJpVcKH+uAOLV2qLqiyla128dN9TkEWfrYmxG6C2boDe5KcNQqZF3orkqzGgOMvZ/JNekA== +node-domexception@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" + integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== + node-emoji@^1.10.0: version "1.11.0" resolved "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz" @@ -3476,6 +3493,15 @@ node-fetch@^2.6.1, node-fetch@^2.6.7: dependencies: whatwg-url "^5.0.0" +node-fetch@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.3.2.tgz#d1e889bacdf733b4ff3b2b243eb7a12866a0b78b" + integrity sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA== + dependencies: + data-uri-to-buffer "^4.0.0" + fetch-blob "^3.1.4" + formdata-polyfill "^4.0.10" + nodesplash@^0.1.1: version "0.1.1" resolved "https://registry.npmjs.org/nodesplash/-/nodesplash-0.1.1.tgz" @@ -3911,6 +3937,18 @@ pkg-dir@^5.0.0: dependencies: find-up "^5.0.0" +play-audio@^0.5.2: + version "0.5.2" + resolved "https://registry.yarnpkg.com/play-audio/-/play-audio-0.5.2.tgz#191dd45b95ff64ae20bbd718671da4e987d4edd1" + integrity sha512-ZAqHUKkQLix2Iga7pPbsf1LpUoBjcpwU93F1l3qBIfxYddQLhxS6GKmS0d3jV8kSVaUbr6NnOEcEMFvuX93SWQ== + +play-dl@^1.9.7: + version "1.9.7" + resolved "https://registry.yarnpkg.com/play-dl/-/play-dl-1.9.7.tgz#885beb66ad3b450632733240faeb65040a43b30f" + integrity sha512-KpgerWxUCY4s9Mhze2qdqPhiqd8Ve6HufpH9mBH3FN+vux55qSh6WJKDabfie8IBHN7lnrAlYcT/UdGax58c2A== + dependencies: + play-audio "^0.5.2" + please-upgrade-node@^3.2.0: version "3.2.0" resolved "https://registry.npmjs.org/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz" @@ -4317,11 +4355,6 @@ safe-regex-test@^1.0.3: resolved "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== -sax@^1.1.3, sax@^1.2.4: - version "1.2.4" - resolved "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz" - integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== - semver-compare@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz" @@ -5059,6 +5092,11 @@ wcwidth@^1.0.1: dependencies: defaults "^1.0.3" +web-streams-polyfill@^3.0.3: + version "3.3.3" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" + integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw== + webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz" @@ -5228,15 +5266,6 @@ yocto-queue@^1.0.0: resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz" integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g== -ytdl-core@^4.11.5: - version "4.11.5" - resolved "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.11.5.tgz" - integrity sha512-27LwsW4n4nyNviRCO1hmr8Wr5J1wLLMawHCQvH8Fk0hiRqrxuIu028WzbJetiYH28K8XDbeinYW4/wcHQD1EXA== - dependencies: - m3u8stream "^0.8.6" - miniget "^4.2.2" - sax "^1.1.3" - ytsr@^3.8.4: version "3.8.4" resolved "https://registry.npmjs.org/ytsr/-/ytsr-3.8.4.tgz"