Merge pull request #311 from Xonshiz/xonshi-2022-04-19

Added support for :
- mangatoon.mobi [#246 ]
- mangadex.org [#195 ]

Changelog.md

@@ -128,4 +128,6 @@
 - Added support for webtoons.com (No audio download yet) [Fix for #284] [2021.09.05.1]
 - Fix for #299 [2022.04.16]
 - Added support for TMOFans/lectortmo.com [Fix for #231] [2022.04.17]
-- Fix for #236 [2022.04.17]
+- Fix for #236 [2022.04.17]
+- Added support for mangatoon.mobi [Fix for #246] [2022.04.19]
+- Added support for mangadex.org [Fix for #195] [2022.04.19]

Supported_Sites.md

@@ -25,4 +25,6 @@
 * [readcomics.io](https://www.readcomics.io/)
 * [otakusmash.com](http://www.otakusmash.com/)
 * [webtoons.com](https://www.webtoons.com/)
-* [TMOFans/lectortmo.com](https://lectortmo.com/)
+* [TMOFans/lectortmo.com](https://lectortmo.com/)
+* [Mangatoon.mobi](https://mangatoon.mobi)
+* [Mangadex.org](https://mangadex.org/)

comic_dl/__version__.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__version__ = "2022.04.17"
+__version__ = "2022.04.19"

comic_dl/globalFunctions.py

@@ -276,8 +276,12 @@ def worker():
             pbar.close()
 
     @staticmethod
-    def create_file_directory(chapter_number, comic_name):
-        comic = re.sub('[^\w\-_. \[\]]', '-', str(comic_name))
+    def create_file_directory(chapter_number, comic_name, dynamic_sub=None):
+        comic = comic_name
+        if dynamic_sub:
+            comic = re.sub(rf'[^\w\-_. \[\]{dynamic_sub}]', '-', str(comic_name))
+        else:
+            comic = re.sub('[^\w\-_. \[\]]', '-', str(comic_name))
         chapter = re.sub('[^\w\-_. \[\]]', '-', str(chapter_number))
         file_directory = comic + os.sep + chapter + os.sep
         return file_directory

comic_dl/honcho.py

@@ -27,6 +27,8 @@
 from .sites import manganelo
 from .sites import webtoons
 from .sites import lectortmo
+from .sites import mangatoonMobi
+from .sites import mangadex
 
 
 class Honcho(object):
@@ -249,5 +251,17 @@ def checker(self, comic_url, download_directory, chapter_range, **kwargs):
                                 chapter_range=chapter_range, conversion=kwargs.get("conversion"),
                                 keep_files=kwargs.get("keep_files"),
                                 print_index=print_index)
+        elif domain in ["www.mangatoon.mobi", "mangatoon.mobi"]:
+            mangatoonMobi.MangatoonMobi(manga_url=comic_url, logger=logging, current_directory=current_directory,
+                                        sorting_order=sorting, log_flag=log_flag, download_directory=download_directory,
+                                        chapter_range=chapter_range, conversion=kwargs.get("conversion"),
+                                        keep_files=kwargs.get("keep_files"),
+                                        print_index=print_index)
+        elif domain in ["www.mangadex.org", "mangadex.org"]:
+            mangadex.Mangadex(manga_url=comic_url, logger=logging, current_directory=current_directory,
+                              sorting_order=sorting, log_flag=log_flag, download_directory=download_directory,
+                              chapter_range=chapter_range, conversion=kwargs.get("conversion"),
+                              keep_files=kwargs.get("keep_files"),
+                              print_index=print_index)
         else:
             print("%s is not supported at the moment. You can request it on the Github repository." % domain)

comic_dl/sites/lectortmo.py

@@ -3,10 +3,7 @@
 import random
 import sys
 
-from bs4 import BeautifulSoup
-
 from comic_dl import globalFunctions
-import re
 import os
 import logging
 import json

comic_dl/sites/mangadex.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from comic_dl import globalFunctions
+import os
+import logging
+import json
+
+
+class Mangadex(object):
+    def __init__(self, manga_url, download_directory, chapter_range, **kwargs):
+
+        current_directory = kwargs.get("current_directory")
+        conversion = kwargs.get("conversion")
+        keep_files = kwargs.get("keep_files")
+        self.logging = kwargs.get("log_flag")
+        self.sorting = kwargs.get("sorting_order")
+        self.comic_name = None
+        self.print_index = kwargs.get("print_index")
+        # https://mangadex.org/chapter/17fd3a89-605d-4f6a-93be-71e559e0889c/4
+        if "/chapter/" in manga_url:
+            self.single_chapter(manga_url, self.comic_name, download_directory, conversion=conversion,
+                                keep_files=keep_files)
+        else:
+            # https://mangadex.org/title/19465f6a-1c11-4179-891e-68293402b883/seton-academy-join-the-pack
+            self.full_series(manga_url, self.comic_name, self.sorting, download_directory, chapter_range=chapter_range,
+                             conversion=conversion, keep_files=keep_files)
+
+    def single_chapter(self, comic_url, comic_name, download_directory, conversion, keep_files, volume=None):
+        comic_url = str(comic_url)
+        chapter_split = comic_url.split('/')
+        chapter_id = chapter_split[-2] if len(chapter_split) > 5 else chapter_split[-1]
+        links = []
+        file_names = []
+        chapter_number = chapter_id
+        # Get image info
+        # https://api.mangadex.org/at-home/server/17fd3a89-605d-4f6a-93be-71e559e0889c?forcePort443=false
+        api_images = "https://api.mangadex.org/at-home/server/{0}?forcePort443=false".format(chapter_id)
+        source_image_list, cookies = globalFunctions.GlobalFunctions().page_downloader(manga_url=api_images)
+        # Get chapter info
+        api_chapter_info = "https://api.mangadex.org/chapter/{0}?includes[]=scanlation_group&includes[]=manga&includes[]=user".format(chapter_id)
+        source_chapter_info, cookies = globalFunctions.GlobalFunctions().page_downloader(manga_url=api_chapter_info)
+        source_chapter_info = json.loads(str(source_chapter_info))
+        image_info = json.loads(str(source_image_list))
+        base_image_url = image_info['baseUrl']
+        base_hash = image_info['chapter']['hash']
+        images = image_info['chapter']['data']
+        for idx, image in enumerate(images):
+            links.append("{0}/data/{1}/{2}".format(base_image_url, base_hash, image))
+            img_extension = str(image).rsplit('.', 1)[-1]
+            file_names.append('{0}.{1}'.format(idx, img_extension))
+        if source_chapter_info:
+            chapter_number = source_chapter_info['data']['attributes']['chapter']
+            for relation in source_chapter_info['data']['relationships']:
+                if self.comic_name:
+                    break
+                if relation['type'] == "manga":
+                    try:
+                        self.comic_name = relation['attributes']['title']['en']
+                        break
+                    except Exception as NameNotFound:
+                        dict_obj = dict(relation['attributes']['title'])
+                        for key in dict_obj.keys():
+                            self.comic_name = dict_obj[key]
+                            # We'll take the first one that comes and break out of this loop
+                            break
+        else:
+            self.comic_name = chapter_id
+        file_directory = globalFunctions.GlobalFunctions().create_file_directory(chapter_number, self.comic_name)
+        if volume:
+            file_directory = file_directory.rsplit(os.sep, 2)[0]
+            file_directory = globalFunctions.GlobalFunctions().create_file_directory(chapter_number, file_directory + os.sep + volume, os.sep)
+        directory_path = os.path.realpath(str(download_directory) + "/" + str(file_directory))
+
+        if not os.path.exists(directory_path):
+            os.makedirs(directory_path)
+        globalFunctions.GlobalFunctions().multithread_download(chapter_number, self.comic_name, comic_url,
+                                                               directory_path,
+                                                               file_names, links, self.logging)
+
+        globalFunctions.GlobalFunctions().conversion(directory_path, conversion, keep_files, self.comic_name,
+                                                     chapter_number)
+
+        return 0
+
+    def full_series(self, comic_url, comic_name, sorting, download_directory, chapter_range, conversion, keep_files):
+        comic_id = str(comic_url).rsplit('/', 2)[-2]
+        comic_detail_url = "https://api.mangadex.org/manga/{0}/aggregate?translatedLanguage[]=en".format(comic_id)
+        source, cookies = globalFunctions.GlobalFunctions().page_downloader(manga_url=comic_detail_url)
+        source = json.loads(str(source))
+
+        all_links = []
+        all_volumes = {}
+        volumes = dict(source['volumes'])
+        for volume in volumes.keys():
+            volume_info = dict(volumes[volume])
+            chapters = dict(volume_info.get('chapters', {}))
+            for chapter in chapters.keys():
+                chapter = dict(chapters[chapter])
+                # https://mangadex.org/chapter/17fd3a89-605d-4f6a-93be-71e559e0889c/4
+                chapter_url = "https://mangadex.org/chapter/{0}/{1}".format(chapter.get('id'), chapter.get('chapter', 1))
+                all_links.append(chapter_url)
+                all_volumes[chapter_url] = "Volume {0}".format(volume)
+
+
+
+
+
+        logging.debug("All Links : {0}".format(all_links))
+
+        # Uh, so the logic is that remove all the unnecessary chapters beforehand
+        #  and then pass the list for further operations.
+        if chapter_range != "All":
+            # -1 to shift the episode number accordingly to the INDEX of it. List starts from 0 xD!
+            starting = int(str(chapter_range).split("-")[0]) - 1
+
+            if str(chapter_range).split("-")[1].isdigit():
+                ending = int(str(chapter_range).split("-")[1])
+            else:
+                ending = len(all_links)
+
+            indexes = [x for x in range(starting, ending)]
+
+            all_links = [all_links[x] for x in indexes][::-1]
+        else:
+            all_links = all_links
+
+        if self.print_index:
+            idx = 0
+            for chap_link in all_links:
+                idx = idx + 1
+                print(str(idx) + ": " + chap_link)
+            return
+
+        if str(sorting).lower() in ['new', 'desc', 'descending', 'latest']:
+            for chap_link in all_links:
+                try:
+                    self.single_chapter(comic_url=chap_link, comic_name=comic_name,
+                                        download_directory=download_directory,
+                                        conversion=conversion, keep_files=keep_files,
+                                        volume=all_volumes.get(chap_link))
+                except Exception as ex:
+                    logging.error("Error downloading : %s" % chap_link)
+                    break  # break to continue processing other mangas
+                # if chapter range contains "__EnD__" write new value to config.json
+                # @Chr1st-oo - modified condition due to some changes on automatic download and config.
+                if chapter_range != "All" and (
+                        chapter_range.split("-")[1] == "__EnD__" or len(chapter_range.split("-")) == 3):
+                    globalFunctions.GlobalFunctions().addOne(comic_url)
+        elif str(sorting).lower() in ['old', 'asc', 'ascending', 'oldest', 'a']:
+            # print("Running this")
+            for chap_link in all_links[::-1]:
+                try:
+                    self.single_chapter(comic_url=chap_link, comic_name=comic_name,
+                                        download_directory=download_directory,
+                                        conversion=conversion, keep_files=keep_files,
+                                        volume=all_volumes.get(chap_link))
+                except Exception as ex:
+                    logging.error("Error downloading : %s" % chap_link)
+                    break  # break to continue processing other mangas
+                # if chapter range contains "__EnD__" write new value to config.json
+                # @Chr1st-oo - modified condition due to some changes on automatic download and config.
+                if chapter_range != "All" and (
+                        chapter_range.split("-")[1] == "__EnD__" or len(chapter_range.split("-")) == 3):
+                    globalFunctions.GlobalFunctions().addOne(comic_url)
+
+        return 0
+
+    def extract_image_link_from_html(self, source):
+        image_tags = source.find_all("img", {"class": "viewer-image viewer-page"})
+        img_link = None
+        for element in image_tags:
+            img_link = element['src']
+        return img_link

comic_dl/sites/mangatoonMobi.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from comic_dl import globalFunctions
+import os
+import logging
+
+
+class MangatoonMobi(object):
+    def __init__(self, manga_url, download_directory, chapter_range, **kwargs):
+
+        current_directory = kwargs.get("current_directory")
+        conversion = kwargs.get("conversion")
+        keep_files = kwargs.get("keep_files")
+        self.logging = kwargs.get("log_flag")
+        self.sorting = kwargs.get("sorting_order")
+        self.comic_name = None
+        self.print_index = kwargs.get("print_index")
+        # https://mangatoon.mobi/en/watch/1632209/114816
+        if "/watch/" in manga_url:
+            self.single_chapter(manga_url, self.comic_name, download_directory, conversion=conversion,
+                                keep_files=keep_files)
+        else:
+            # https://mangatoon.mobi/en/the-call-animals?content_id=1632209
+            self.full_series(manga_url, self.comic_name, self.sorting, download_directory, chapter_range=chapter_range,
+                             conversion=conversion, keep_files=keep_files)
+
+    def single_chapter(self, comic_url, comic_name, download_directory, conversion, keep_files):
+        comic_url = str(comic_url)
+        # https://mangatoon.mobi/en/watch/1632209/114816
+        chapter_number = comic_url.rsplit('/', 1)[-1]
+        links = []
+        file_names = []
+        source, cookies = globalFunctions.GlobalFunctions().page_downloader(manga_url=comic_url)
+        title = source.find_all("div", {"class": "title"})
+        if len(title) > 0:
+            self.comic_name = title[0].text.strip()
+        else:
+            self.comic_name = comic_url.rsplit('/', 2)[-2]
+        image_holder_divs = source.find_all("div", {"style": "position: relative;"})
+        if len(image_holder_divs) > 0:
+            for idx, img_tag in enumerate(image_holder_divs):
+                x = img_tag.findAll('img')
+                for a in x:
+                    if "/icon/" not in a['src']:
+                        img_url = a['src']
+                        links.append(str(img_url).strip())
+                        img_extension = str(img_url).rsplit('.', 1)[-1]
+                        file_names.append('{0}.{1}'.format(idx, img_extension))
+        file_directory = globalFunctions.GlobalFunctions().create_file_directory(chapter_number, self.comic_name)
+
+        directory_path = os.path.realpath(str(download_directory) + "/" + str(file_directory))
+
+        if not os.path.exists(directory_path):
+            os.makedirs(directory_path)
+        globalFunctions.GlobalFunctions().multithread_download(chapter_number, self.comic_name, comic_url,
+                                                               directory_path,
+                                                               file_names, links, self.logging)
+
+        globalFunctions.GlobalFunctions().conversion(directory_path, conversion, keep_files, self.comic_name,
+                                                     chapter_number)
+
+        return 0
+
+    def full_series(self, comic_url, comic_name, sorting, download_directory, chapter_range, conversion, keep_files):
+        source, cookies = globalFunctions.GlobalFunctions().page_downloader(manga_url=comic_url)
+
+        all_links = []
+        all_chapter_links = source.find_all("a", {"class": "episode-item-new"})
+        for chapter in all_chapter_links:
+            chapter_url = "https://mangatoon.mobi{0}".format(chapter['href'])
+            all_links.append(chapter_url)
+
+        logging.debug("All Links : {0}".format(all_links))
+
+        # Uh, so the logic is that remove all the unnecessary chapters beforehand
+        #  and then pass the list for further operations.
+        if chapter_range != "All":
+            # -1 to shift the episode number accordingly to the INDEX of it. List starts from 0 xD!
+            starting = int(str(chapter_range).split("-")[0]) - 1
+
+            if str(chapter_range).split("-")[1].isdigit():
+                ending = int(str(chapter_range).split("-")[1])
+            else:
+                ending = len(all_links)
+
+            indexes = [x for x in range(starting, ending)]
+
+            all_links = [all_links[x] for x in indexes][::-1]
+        else:
+            all_links = all_links
+
+        if self.print_index:
+            idx = 0
+            for chap_link in all_links:
+                idx = idx + 1
+                print(str(idx) + ": " + chap_link)
+            return
+
+        if str(sorting).lower() in ['new', 'desc', 'descending', 'latest']:
+            for chap_link in all_links:
+                try:
+                    self.single_chapter(comic_url=chap_link, comic_name=comic_name,
+                                        download_directory=download_directory,
+                                        conversion=conversion, keep_files=keep_files)
+                except Exception as ex:
+                    logging.error("Error downloading : %s" % chap_link)
+                    break  # break to continue processing other mangas
+                # if chapter range contains "__EnD__" write new value to config.json
+                # @Chr1st-oo - modified condition due to some changes on automatic download and config.
+                if chapter_range != "All" and (
+                        chapter_range.split("-")[1] == "__EnD__" or len(chapter_range.split("-")) == 3):
+                    globalFunctions.GlobalFunctions().addOne(comic_url)
+        elif str(sorting).lower() in ['old', 'asc', 'ascending', 'oldest', 'a']:
+            # print("Running this")
+            for chap_link in all_links[::-1]:
+                try:
+                    self.single_chapter(comic_url=chap_link, comic_name=comic_name,
+                                        download_directory=download_directory,
+                                        conversion=conversion, keep_files=keep_files)
+                except Exception as ex:
+                    logging.error("Error downloading : %s" % chap_link)
+                    break  # break to continue processing other mangas
+                # if chapter range contains "__EnD__" write new value to config.json
+                # @Chr1st-oo - modified condition due to some changes on automatic download and config.
+                if chapter_range != "All" and (
+                        chapter_range.split("-")[1] == "__EnD__" or len(chapter_range.split("-")) == 3):
+                    globalFunctions.GlobalFunctions().addOne(comic_url)
+
+        return 0
+
+    def extract_image_link_from_html(self, source):
+        image_tags = source.find_all("img", {"class": "viewer-image viewer-page"})
+        img_link = None
+        for element in image_tags:
+            img_link = element['src']
+        return img_link