Migrate the trending page to YouTube.js (#3005)

* Migrate the trending page to YouTube.js

* Move more of the logic to the local API file

* This function doesn't need to be exported anymore

package.json

@@ -50,7 +50,6 @@
     "@fortawesome/vue-fontawesome": "^2.0.9",
     "@freetube/youtube-chat": "^1.1.2",
     "@freetube/yt-comment-scraper": "^6.2.0",
-    "@freetube/yt-trending-scraper": "^3.1.1",
     "@silvermine/videojs-quality-selector": "^1.2.5",
     "autolinker": "^4.0.0",
     "browserify": "^17.0.0",

src/renderer/helpers/api/local.js

@@ -2,7 +2,7 @@ import { Innertube, Session } from 'youtubei.js'
 import { join } from 'path'
 
 import { PlayerCache } from './PlayerCache'
-import { getUserDataPath } from '../utils'
+import { extractNumberFromString, getUserDataPath } from '../utils'
 
 /**
  * Creates a lightweight Innertube instance, which is faster to create or
@@ -12,21 +12,24 @@ import { getUserDataPath } from '../utils'
  * 1. the request for the session
  * 2. fetch a page that contains a link to the player
  * 3. if the player isn't cached, it is downloaded and transformed
- * @param {boolean} withPlayer set to true to get an Innertube instance that can decode the streaming URLs
+ * @param {object} options
+ * @param {boolean} options.withPlayer set to true to get an Innertube instance that can decode the streaming URLs
+ * @param {string|undefined} options.location the geolocation to pass to YouTube get different content
  * @returns the Innertube instance
  */
-async function createInnertube(withPlayer = false) {
-  if (withPlayer) {
+async function createInnertube(options = { withPlayer: false, location: undefined }) {
+  if (options.withPlayer) {
     const userData = await getUserDataPath()
 
     return await Innertube.create({
       // use browser fetch
+      location: options.location,
       fetch: (input, init) => fetch(input, init),
       cache: new PlayerCache(join(userData, 'player_cache'))
     })
   } else {
     // from https://github.com/LuanRT/YouTube.js/pull/240
-    const sessionData = await Session.getSessionData()
+    const sessionData = await Session.getSessionData(undefined, options.location)
 
     const session = new Session(
       sessionData.context,
@@ -62,6 +65,35 @@ export async function getLocalPlaylist(id) {
   return await innertube.getPlaylist(id)
 }
 
+/**
+ * @typedef {import('youtubei.js/dist/src/core/TabbedFeed').default} TabbedFeed
+ */
+
+/**
+ * @param {string} location
+ * @param {string} tab
+ * @param {TabbedFeed|null} instance
+ */
+export async function getLocalTrending(location, tab, instance) {
+  if (instance === null) {
+    const innertube = await createInnertube({ location })
+    instance = await innertube.getTrending()
+  }
+
+  // youtubei.js's tab names are localised, so we need to use the index to get tab name that youtubei.js expects
+  const tabIndex = ['default', 'music', 'gaming', 'movies'].indexOf(tab)
+  const resultsInstance = await instance.getTabByName(instance.tabs[tabIndex])
+
+  const results = resultsInstance.videos
+    .filter((video) => video.type === 'Video')
+    .map(parseLocalListVideo)
+
+  return {
+    results,
+    instance: resultsInstance
+  }
+}
+
 /**
  * @typedef {import('youtubei.js/dist/src/parser/classes/PlaylistVideo').default} PlaylistVideo
  */
@@ -78,3 +110,25 @@ export function parseLocalPlaylistVideo(video) {
     lengthSeconds: isNaN(video.duration.seconds) ? '' : video.duration.seconds
   }
 }
+
+/**
+ * @typedef {import('youtubei.js/dist/src/parser/classes/Video').default} Video
+ */
+
+/**
+ * @param {Video} video
+ */
+function parseLocalListVideo(video) {
+  return {
+    type: 'video',
+    videoId: video.id,
+    title: video.title.text,
+    author: video.author.name,
+    authorId: video.author.id,
+    description: video.description,
+    viewCount: extractNumberFromString(video.view_count.text),
+    publishedText: video.published.text,
+    lengthSeconds: isNaN(video.duration.seconds) ? '' : video.duration.seconds,
+    liveNow: video.is_live
+  }
+}

src/renderer/views/Search/Search.js

@@ -3,7 +3,7 @@ import { mapActions } from 'vuex'
 import FtLoader from '../../components/ft-loader/ft-loader.vue'
 import FtCard from '../../components/ft-card/ft-card.vue'
 import FtElementList from '../../components/ft-element-list/ft-element-list.vue'
-import { calculateLengthInSeconds } from '@freetube/yt-trending-scraper/src/HtmlParser'
+import { timeToSeconds } from 'youtubei.js/dist/src/utils/Utils'
 import { copyToClipboard, searchFiltersMatch, showToast } from '../../helpers/utils'
 
 export default Vue.extend({
@@ -155,7 +155,7 @@ export default Vue.extend({
             let videoDuration = video.duration
             const videoId = video.id
             if (videoDuration !== null && videoDuration !== '' && videoDuration !== 'LIVE' && videoDuration !== 'UPCOMING' && videoDuration !== 'PREMIERE') {
-              videoDuration = calculateLengthInSeconds(video.duration)
+              videoDuration = timeToSeconds(video.duration)
             }
             dataToShow.push(
               {

src/renderer/views/Trending/Trending.js

@@ -6,8 +6,8 @@ import FtElementList from '../../components/ft-element-list/ft-element-list.vue'
 import FtIconButton from '../../components/ft-icon-button/ft-icon-button.vue'
 import FtFlexBox from '../../components/ft-flex-box/ft-flex-box.vue'
 
-import { scrapeTrendingPage } from '@freetube/yt-trending-scraper'
 import { copyToClipboard, showToast } from '../../helpers/utils'
+import { getLocalTrending } from '../../helpers/api/local'
 
 export default Vue.extend({
   name: 'Trending',
@@ -22,7 +22,8 @@ export default Vue.extend({
     return {
       isLoading: false,
       shownResults: [],
-      currentTab: 'default'
+      currentTab: 'default',
+      trendingInstance: null
     }
   },
   computed: {
@@ -77,27 +78,21 @@ export default Vue.extend({
       }
     },
 
-    getTrendingInfoLocal: function () {
+    getTrendingInfoLocal: async function () {
       this.isLoading = true
 
-      const param = {
-        parseCreatorOnRise: false,
-        page: this.currentTab,
-        geoLocation: this.region
-      }
-
-      scrapeTrendingPage(param).then((result) => {
-        const returnData = result.filter((item) => {
-          return item.type === 'video' || item.type === 'channel' || item.type === 'playlist'
-        })
+      try {
+        const { results, instance } = await getLocalTrending(this.region, this.currentTab, this.trendingInstance)
 
-        this.shownResults = returnData
+        this.shownResults = results
         this.isLoading = false
-        this.$store.commit('setTrendingCache', { value: returnData, page: this.currentTab })
+        this.trendingInstance = instance
+
+        this.$store.commit('setTrendingCache', { value: results, page: this.currentTab })
         setTimeout(() => {
           this.$refs[this.currentTab].focus()
         })
-      }).catch((err) => {
+      } catch (err) {
         console.error(err)
         const errorMessage = this.$t('Local API Error (Click to copy)')
         showToast(`${errorMessage}: ${err}`, 10000, () => {
@@ -109,7 +104,7 @@ export default Vue.extend({
         } else {
           this.isLoading = false
         }
-      })
+      }
     },
 
     getTrendingInfoCache: function () {

yarn.lock

@@ -1066,13 +1066,6 @@
   dependencies:
     axios "^0.27.2"
 
-"@freetube/yt-trending-scraper@^3.1.1":
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/@freetube/yt-trending-scraper/-/yt-trending-scraper-3.1.1.tgz#82d11ac3dad3ea25bc0f30d68e315364e9977046"
-  integrity sha512-qgWFLefcKiLfJmETtEeDVuv9MQ50JFwkGlR3NITDG/SzdrhTFBizG5Ggs34sub6UQd6M6Ib8HxI/lgtKYqN7qA==
-  dependencies:
-    axios "^0.27.2"
-
 "@humanwhocodes/config-array@^0.5.0":
   version "0.5.0"
   resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.5.0.tgz#1407967d4c6eecd7388f83acf1eaf4d0c6e58ef9"