From 7f1c291408a3e624f7b4e889b632981f7fdfaa28 Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Sun, 4 Dec 2022 16:47:00 +0900 Subject: [PATCH 1/2] fix `TwitterCrawler` to extract only videos --- twitter_video_tools/twitter_crawler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/twitter_video_tools/twitter_crawler.py b/twitter_video_tools/twitter_crawler.py index 2fa504e..92ab43a 100644 --- a/twitter_video_tools/twitter_crawler.py +++ b/twitter_video_tools/twitter_crawler.py @@ -72,7 +72,7 @@ def get_recent_liked_tweet(self, username: str) -> str: self._goto_liked_tweets(username) return self._get_article_links_in_current_screen()[0] - def get_video_of_tweet(self, link: str, timeout: Optional[float] = 5000) -> list[tuple[str, str]]: + def get_video_of_tweet(self, link: str, timeout: Optional[float] = 10000) -> list[tuple[str, str]]: video_links: list[str] = [] def _request_m3u8_capture_handler(request: Request) -> None: @@ -101,7 +101,7 @@ def _get_article_links_in_current_screen(self) -> list[str]: links: list[str] = [] while True: - articles = self.page.locator('article') + articles = self.page.locator('article:has(video)') article_length = articles.count() try: links = [ @@ -112,5 +112,6 @@ def _get_article_links_in_current_screen(self) -> list[str]: break except Error: # if articles in the page are not reachable self.page.mouse.wheel(0, 500) # scrolling down to refresh the articles + self.page.mouse.wheel(0, -500) # scrolling down to refresh the articles return links From a1f5362c5fd7dcb8cb29facee55defbed6b25f40 Mon Sep 17 00:00:00 2001 From: YeonGyu-Kim Date: Sun, 4 Dec 2022 16:51:25 +0900 Subject: [PATCH 2/2] Update packages version --- poetry.lock | 20 ++++++++++---------- pyproject.toml | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index bf5e52d..61ae775 100644 --- a/poetry.lock +++ b/poetry.lock @@ -362,14 +362,14 @@ typing-extensions = "*" [[package]] name = "pylint" -version = "2.15.6" +version = "2.15.7" description = "python code static checker" category = "dev" optional = false python-versions = ">=3.7.2" [package.dependencies] -astroid = ">=2.12.12,<=2.14.0-dev0" +astroid = ">=2.12.13,<=2.14.0-dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} dill = ">=0.2" isort = ">=4.2.5,<6" @@ -407,7 +407,7 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pyright" -version = "1.1.281" +version = "1.1.282" description = "Command line wrapper for pyright" category = "dev" optional = false @@ -536,7 +536,7 @@ python-versions = "*" [[package]] name = "types-invoke" -version = "1.7.3.15" +version = "1.7.3.16" description = "Typing stubs for invoke" category = "dev" optional = false @@ -1171,8 +1171,8 @@ pyee = [ {file = "pyee-9.0.4.tar.gz", hash = "sha256:2770c4928abc721f46b705e6a72b0c59480c4a69c9a83ca0b00bb994f1ea4b32"}, ] pylint = [ - {file = "pylint-2.15.6-py3-none-any.whl", hash = "sha256:15060cc22ed6830a4049cf40bc24977744df2e554d38da1b2657591de5bcd052"}, - {file = "pylint-2.15.6.tar.gz", hash = "sha256:25b13ddcf5af7d112cf96935e21806c1da60e676f952efb650130f2a4483421c"}, + {file = "pylint-2.15.7-py3-none-any.whl", hash = "sha256:1d561d1d3e8be9dd880edc685162fbdaa0409c88b9b7400873c0cf345602e326"}, + {file = "pylint-2.15.7.tar.gz", hash = "sha256:91e4776dbcb4b4d921a3e4b6fec669551107ba11f29d9199154a01622e460a57"}, ] pylint-quotes = [ {file = "pylint-quotes-0.2.3.tar.gz", hash = "sha256:2d6bb3fa8a1a85af3af8a0ca875a719ac5bcdb735c45756284699d809c109c95"}, @@ -1183,8 +1183,8 @@ pyparsing = [ {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] pyright = [ - {file = "pyright-1.1.281-py3-none-any.whl", hash = "sha256:9e52d460c5201c8870a3aa053289a0595874b739314d67b0fdf531c25f17ca01"}, - {file = "pyright-1.1.281.tar.gz", hash = "sha256:21be27aa562d3e2f1563515a118dcf9a3fd197747604ed47277d36cfe5376c7a"}, + {file = "pyright-1.1.282-py3-none-any.whl", hash = "sha256:f899fda0a6449d81687ea0e1ad5b25e268845b8a0c98080ada93a6050d9696b8"}, + {file = "pyright-1.1.282.tar.gz", hash = "sha256:ec615b1e4242e5657bb69f46bbc1cc600d0628bc7fed988bfd9d1c8e0257eb4b"}, ] pytest = [ {file = "pytest-7.2.0-py3-none-any.whl", hash = "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71"}, @@ -1265,8 +1265,8 @@ types-beautifulsoup4 = [ {file = "types_beautifulsoup4-4.11.6.1-py3-none-any.whl", hash = "sha256:c1f803367a2b07ad4fdac40ddbea557010dc4ddd1ee92d801f317eb02e2e3c72"}, ] types-invoke = [ - {file = "types-invoke-1.7.3.15.tar.gz", hash = "sha256:25de028906fda7f7aa2dc30fec02769f9194a8f9ecaad1abed55b1408bdc6d53"}, - {file = "types_invoke-1.7.3.15-py3-none-any.whl", hash = "sha256:0ad12cde503ae5ea63a3c167c90bf171087954dfb9c170b879b8325a883ad4ab"}, + {file = "types-invoke-1.7.3.16.tar.gz", hash = "sha256:a1f8158db43d074ebd741a2bc9f403947272663bf634c4a0935fc6a5d7646448"}, + {file = "types_invoke-1.7.3.16-py3-none-any.whl", hash = "sha256:875ef44186136099bf5c430aed8c5d8ccc5107d77dc2390666844a48142b573d"}, ] types-requests = [ {file = "types-requests-2.28.11.5.tar.gz", hash = "sha256:a7df37cc6fb6187a84097da951f8e21d335448aa2501a6b0a39cbd1d7ca9ee2a"}, diff --git a/pyproject.toml b/pyproject.toml index fdaa40e..9031de1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ yt-dlp = "^2022.11.11" monkeytype = "^22.2.0" mypy = "^0.982" yapf = "^0.32.0" -pylint = "^2.15.5" +pylint = "^2.15.7" isort = "^5.10.1" pytest = "^7.2.0" pytest-cov = "^4.0.0" @@ -77,7 +77,7 @@ types-requests = "^2.28.11" types-beautifulsoup4 = "^4.11.6" types-invoke = "^1.7.3" types-toml = "^0.10.8" -pyright = "^1.1.280" +pyright = "^1.1.282" colorama = "^0.4.6" [tool.pylint.MASTER]