Added Search feature to Youtube Scraper (#194)

* Added search functionality to YoutubeSkraper.kt and test in YoutubeSkraperTest.kt * Changes to adapt library structure.
sokomishalov · Aug 25, 2021 · acfb307 · acfb307
1 parent e323b70
commit acfb307
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/skrapers/src/main/kotlin/ru/sokomishalov/skraper/provider/youtube/YoutubeSkraper.kt b/skrapers/src/main/kotlin/ru/sokomishalov/skraper/provider/youtube/YoutubeSkraper.kt
@@ -53,9 +53,11 @@ open class YoutubeSkraper @JvmOverloads constructor(
 
         val jsonMetadata = page?.readJsonMetadata()
 
+        val fieldName = if(path.startsWith(SEARCH_PREFIX)) "videoRenderer" else "gridVideoRenderer"
+
         val rawPosts = jsonMetadata
-            ?.findParents("gridVideoRenderer")
-            ?.map { it["gridVideoRenderer"] }
+            ?.findParents(fieldName)
+            ?.map { it[fieldName] }
             .orEmpty()
 
         emitBatch(rawPosts) {
@@ -64,7 +66,7 @@ open class YoutubeSkraper @JvmOverloads constructor(
                 text = getString("title.runs.0.text"),
                 publishedAt = getString("publishedTimeText")?.extractTimeAgo(),
                 statistics = PostStatistics(
-                    views = getString("viewCountText.simpleText")?.substringBefore(" ")?.toIntOrNull(),
+                    views = getString("viewCountText.simpleText")?.substringBefore(" ")?.replace(",","")?.toIntOrNull(),
                 ),
                 media = extractVideos()
             )
@@ -205,5 +207,6 @@ open class YoutubeSkraper @JvmOverloads constructor(
 
     companion object {
         const val BASE_URL: String = "https://www.youtube.com"
+        const val SEARCH_PREFIX: String = "/results"
     }
 }
diff --git a/...pers/src/main/kotlin/ru/sokomishalov/skraper/provider/youtube/YoutubeSkraperExtensions.kt b/...pers/src/main/kotlin/ru/sokomishalov/skraper/provider/youtube/YoutubeSkraperExtensions.kt
@@ -30,4 +30,8 @@ fun YoutubeSkraper.getUserPosts(username: String): Flow<Post> {
 
 suspend fun YoutubeSkraper.getUserInfo(username: String): PageInfo? {
     return getPageInfo(path = "/user/${username}")
+}
+
+fun YoutubeSkraper.getSearchPosts(keyword: String): Flow<Post> {
+    return getPosts(path = "/results?search_query=${keyword.replace(" ","+")}")
 }
diff --git a/skrapers/src/test/kotlin/ru/sokomishalov/skraper/provider/youtube/YoutubeSkraperTest.kt b/skrapers/src/test/kotlin/ru/sokomishalov/skraper/provider/youtube/YoutubeSkraperTest.kt
@@ -45,4 +45,9 @@ class YoutubeSkraperTest : SkraperTck() {
     fun `Check media downloading`() {
         assertMediaDownloaded(Video("https://youtu.be/fjUO7xaUHJQ"))
     }
+
+    @Test
+    fun `Check search posts`() {
+        assertPosts { skraper.getSearchPosts("Nothing Else Matters") }
+    }
 }