From ecee017510a3d28f0112c635e95405a81ac0d275 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Wed, 3 Jun 2020 20:33:59 +0800 Subject: [PATCH 01/23] chg: doc: year --- setup.py | 2 +- we_get/__init__.py | 2 +- we_get/core/commands.py | 2 +- we_get/core/completer.py | 2 +- we_get/core/module.py | 2 +- we_get/core/shell.py | 2 +- we_get/core/style.py | 2 +- we_get/core/utils.py | 2 +- we_get/core/we_get.py | 2 +- we_get/modules/1337x.py | 2 +- we_get/modules/__init__.py | 2 +- we_get/modules/eztv.py | 2 +- we_get/modules/the_pirate_bay.py | 2 +- we_get/modules/yts.py | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 1a64e2f..5700e1e 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying permission """ diff --git a/we_get/__init__.py b/we_get/__init__.py index f5b7a3a..a2b45f3 100644 --- a/we_get/__init__.py +++ b/we_get/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying permission """ diff --git a/we_get/core/commands.py b/we_get/core/commands.py index 33efe71..d37c133 100644 --- a/we_get/core/commands.py +++ b/we_get/core/commands.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying. """ diff --git a/we_get/core/completer.py b/we_get/core/completer.py index 6f9284a..ff0f567 100644 --- a/we_get/core/completer.py +++ b/we_get/core/completer.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying. """ diff --git a/we_get/core/module.py b/we_get/core/module.py index d355322..9ec885a 100644 --- a/we_get/core/module.py +++ b/we_get/core/module.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying. """ diff --git a/we_get/core/shell.py b/we_get/core/shell.py index 82fce65..9b77118 100644 --- a/we_get/core/shell.py +++ b/we_get/core/shell.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying. """ import logging diff --git a/we_get/core/style.py b/we_get/core/style.py index 3a3c525..78914e5 100644 --- a/we_get/core/style.py +++ b/we_get/core/style.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying. This code is from the prompt_toolkit library by `Jonathan Slenders`. diff --git a/we_get/core/utils.py b/we_get/core/utils.py index bbae7f1..7d3cf64 100644 --- a/we_get/core/utils.py +++ b/we_get/core/utils.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying. """ diff --git a/we_get/core/we_get.py b/we_get/core/we_get.py index a4da5a0..b2931c2 100644 --- a/we_get/core/we_get.py +++ b/we_get/core/we_get.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying. """ diff --git a/we_get/modules/1337x.py b/we_get/modules/1337x.py index 18308d5..ee99213 100644 --- a/we_get/modules/1337x.py +++ b/we_get/modules/1337x.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying permission """ diff --git a/we_get/modules/__init__.py b/we_get/modules/__init__.py index c105ec2..67ea5b9 100644 --- a/we_get/modules/__init__.py +++ b/we_get/modules/__init__.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying permission """ pass diff --git a/we_get/modules/eztv.py b/we_get/modules/eztv.py index 78e2142..ad9e953 100644 --- a/we_get/modules/eztv.py +++ b/we_get/modules/eztv.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying permission """ diff --git a/we_get/modules/the_pirate_bay.py b/we_get/modules/the_pirate_bay.py index f3708e7..585487e 100644 --- a/we_get/modules/the_pirate_bay.py +++ b/we_get/modules/the_pirate_bay.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying permission """ diff --git a/we_get/modules/yts.py b/we_get/modules/yts.py index fec5213..b572fe3 100644 --- a/we_get/modules/yts.py +++ b/we_get/modules/yts.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) +Copyright (c) 2016-2020 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying permission """ From 42a0b6e97041de48d8cd489167eb8728dcde25f7 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Wed, 3 Jun 2020 20:36:49 +0800 Subject: [PATCH 02/23] new: dev: upgrade requirement --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5700e1e..d40436f 100644 --- a/setup.py +++ b/setup.py @@ -20,8 +20,8 @@ install_requires=[ 'colorama', 'docopt', - 'prompt_toolkit>=2.0.3', - 'Pygments>=2.2.0', + 'prompt-toolkit>=3.0.5', + 'Pygments>=2.6.1', ], extras_require={ 'test': ["pytest", "pytest-flake8", 'vcrpy'], }, From 45989313aea6088fb498422951c2ab4555b530fc Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Wed, 3 Jun 2020 20:49:12 +0800 Subject: [PATCH 03/23] new: test: mypy config --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 8ab3ad9..b842253 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,6 @@ flake8-max-line-length = 99 [flake8] max-line-length = 99 + +[mypy] +ignore_missing_imports = True From c553b762dbc3622db07b8467392acbc02abbc374 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Wed, 3 Jun 2020 21:23:22 +0800 Subject: [PATCH 04/23] new: doc: type hint for msg_item --- we_get/core/utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/we_get/core/utils.py b/we_get/core/utils.py index 7d3cf64..2e41bc9 100644 --- a/we_get/core/utils.py +++ b/we_get/core/utils.py @@ -3,14 +3,17 @@ See the file 'LICENSE' for copying. """ -import sys -from random import choice import re -from os import sep -from we_get import __file__ as p +import sys from glob import glob -from colorama import init as colorama_init +from os import sep +from random import choice +from typing import Dict + from colorama import Fore, Style +from colorama import init as colorama_init + +from we_get import __file__ as p colorama_init(autoreset=True) @@ -112,7 +115,7 @@ def msg_info(msg): sys.stdout.write("%s# %s\n" % (Fore.BLUE, msg)) -def msg_item(item, items): +def msg_item(item: str, items: Dict[str, str]): """ msg_item: print item. @item - name. @seeds - number of seeds if any. From b718138d1ab7e51e9fd6d9330bf6bb6baff40c91 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Wed, 3 Jun 2020 22:17:41 +0800 Subject: [PATCH 05/23] new: user: config file and item color --- we_get/core/we_get.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/we_get/core/we_get.py b/we_get/core/we_get.py index b2931c2..8d1ec96 100644 --- a/we_get/core/we_get.py +++ b/we_get/core/we_get.py @@ -3,15 +3,17 @@ See the file 'LICENSE' for copying. """ -from collections import OrderedDict -from docopt import docopt -from importlib import import_module -from json import dumps -from sys import exit import collections +import configparser import itertools import logging import re +from collections import OrderedDict +from importlib import import_module +from json import dumps +from sys import exit + +from docopt import docopt from we_get.core.utils import ( format_help, @@ -19,7 +21,7 @@ msg_err_trace, msg_error, msg_fetching, - msg_info, + msg_info ) __version__ = "1.1.0" @@ -35,6 +37,7 @@ -f --filter= Match text or regular expression in the torrent name. -n --results= Number of results to retrieve. -S --sort-type= Sort torrents by name/seeds [default: seeds]. + -c --config= Load config file. Video options: -q --quality= Try to match quality for the torrent (720p,1080p, ...). @@ -74,6 +77,11 @@ def parse_args(self): self.results_type = 'J' elif arg == "--sort": self.sort_type = self.pargs[arg][0] + elif arg == "--config": + self.config = configparser.ConfigParser() + config_file = self.pargs[arg][0] + with open(config_file) as f: + self.config.read_file(f) def cut_items(self, items, results): """cut_items: show N items. @@ -185,6 +193,7 @@ def run(self, api_mode=False): # we-get will load 50% faster! from we_get.core.shell import Shell self.shell = Shell() + self.shell.item_color.update(self.config['item_color']) self.shell.shell(self.items, self.pargs) From 5d1575b6d92058d06c413b0f346b29edf0292dc9 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Wed, 3 Jun 2020 22:18:27 +0800 Subject: [PATCH 06/23] chg: dev: print item func - change color - add black color --- we_get/core/utils.py | 59 ++++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/we_get/core/utils.py b/we_get/core/utils.py index 2e41bc9..3887616 100644 --- a/we_get/core/utils.py +++ b/we_get/core/utils.py @@ -8,7 +8,7 @@ from glob import glob from os import sep from random import choice -from typing import Dict +from typing import Dict, Optional from colorama import Fore, Style from colorama import init as colorama_init @@ -17,16 +17,26 @@ colorama_init(autoreset=True) +# supported color from colorama 0.4.3 +# BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE COLORS = { - "red": Fore.RED, - "green": Fore.GREEN, + "black": Fore.BLACK, "blue": Fore.BLUE, - "yellow": Fore.YELLOW, "cyan": Fore.CYAN, - "white": Fore.WHITE, + "green": Fore.GREEN, "magenta": Fore.MAGENTA, + "red": Fore.RED, + "white": Fore.WHITE, + "yellow": Fore.YELLOW, +} +ITEM_COLOR_SET = { + 'leeches': 'red', + 'target': 'green', + 'item': 'white', + 'seeds': 'green', + 'user_status': 'green', + 'user_status_vip': 'magenta' } - def format_help(doc, errmsg): """ format_help: fix help message. @@ -115,29 +125,40 @@ def msg_info(msg): sys.stdout.write("%s# %s\n" % (Fore.BLUE, msg)) -def msg_item(item: str, items: Dict[str, str]): +def msg_item( + item: str, + items: Dict[str, str], + item_color: Optional[Dict[str, str]] = None +): """ msg_item: print item. @item - name. - @seeds - number of seeds if any. - @leeches - number of leeches if any. + @items - item status (leeches, seed, target, etc) + @item_color - item color. """ + cset = ITEM_COLOR_SET.copy() + if item_color is not None: + cset.update(item_color) leeches = items['leeches'] seeds = items['seeds'] target = items['target'] + user_status = items.get('user_status', None) + if user_status and user_status is not None: + if user_status == 'vip': + user_status_text = color(cset['user_status_vip'], user_status) + else: + color(cset['user_status'], user_status) + else: + user_status_text = '' text = ( - "%s %s [%s/%s]" % ( - color("green", target), color("white", item), - color("green", seeds), color("red", leeches) + "%s %s [%s/%s] %s" % ( + color(cset['target'], target), + color(cset['item'], item), + color(cset['seeds'], seeds), + color(cset['leeches'], leeches), + user_status_text ) ) - - user_status = items.get('user_status', None) - if user_status and user_status is not None: - c_text = color('green', user_status) \ - if user_status == 'vip' else color('magenta', user_status) - text += ' %s' % c_text - sys.stdout.write(text + '\n') From 10d60703fa48caca5e8e4e56c6b6acda7f353305 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Wed, 3 Jun 2020 22:19:47 +0800 Subject: [PATCH 07/23] new: handle item color on shell --- we_get/core/shell.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/we_get/core/shell.py b/we_get/core/shell.py index 9b77118..d3cc685 100644 --- a/we_get/core/shell.py +++ b/we_get/core/shell.py @@ -6,20 +6,16 @@ import re from json import dumps from sys import stdout -from we_get.core.utils import printc -from we_get.core.utils import printc_raw -from we_get.core.utils import color -from we_get.core.utils import msg_item -from we_get.core.utils import msg_error -from we_get.core.commands import COMMANDS -from we_get.core.completer import WGCompleter -from we_get.core.style import we_get_prompt_style import prompt_toolkit from prompt_toolkit import prompt -from prompt_toolkit.history import InMemoryHistory from prompt_toolkit.auto_suggest import AutoSuggestFromHistory +from prompt_toolkit.history import InMemoryHistory +from we_get.core.commands import COMMANDS +from we_get.core.completer import WGCompleter +from we_get.core.style import we_get_prompt_style +from we_get.core.utils import color, msg_error, msg_item, printc, printc_raw, ITEM_COLOR_SET PROMPT_TOOLKIT_V2 = prompt_toolkit.__version__.split('.')[0] == '2' if PROMPT_TOOLKIT_V2: @@ -35,6 +31,7 @@ def __init__(self): self.pargs = None self.items = None self.show_links = False + self.item_color = ITEM_COLOR_SET.copy() def prompt_usage(self): printc("white", "Usage: help") @@ -57,7 +54,7 @@ def prompt_no_command(self, command): def prompt_show_items(self): for item in self.items: - msg_item(item, self.items[item]) + msg_item(item, self.items[item], self.item_color) def prompt_verify_command(self, command, args): for x in COMMANDS: From 2736560c58721a466d9c686624f6005f6aa8815e Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Wed, 3 Jun 2020 22:20:03 +0800 Subject: [PATCH 08/23] new: test: msg_item --- tests/test_utils.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/test_utils.py diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..b8a7079 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from we_get.core import utils + + +def test_msg_item(): + args = [ + 'Ubuntu.MATE.16.04.2.[MATE][armhf][img.xz][Uzerus]', + { + 'leeches': '2', + 'link': 'magnet:?xt=urn:btih:D0F23C109D8662A3FE9338F75839AF8D57E5D4A9' + '&dn=Ubuntu+MATE+16.04.2+%5BMATE%5D%5Barmhf%5D%5Bimg.xz%5D%5BUzerus%5D', + 'seeds': '260', + 'target': '1337x'} + ] + utils.msg_item(*args) From edbfc4635aa98f316d38806b617091c7e6bccbed Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Wed, 3 Jun 2020 22:20:22 +0800 Subject: [PATCH 09/23] new: user: example config --- config.cfg | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 config.cfg diff --git a/config.cfg b/config.cfg new file mode 100644 index 0000000..35eb67b --- /dev/null +++ b/config.cfg @@ -0,0 +1,10 @@ +[item_color] +# default from we-get 1.1.1 +# supported color from colorama 0.4.3 +# black, red, green, yellow, blue, magenta, cyan, white +# leeches = red +# target = green +# item = white +# seeds = green +# user_status = green +# user_status_vip = magenta From 88b5b21b0f3293c90132010608acd0fa59709e0e Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Thu, 4 Jun 2020 08:05:48 +0800 Subject: [PATCH 10/23] chg: dev: only change when config exist --- we_get/core/we_get.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/we_get/core/we_get.py b/we_get/core/we_get.py index 8d1ec96..5049ba5 100644 --- a/we_get/core/we_get.py +++ b/we_get/core/we_get.py @@ -193,7 +193,8 @@ def run(self, api_mode=False): # we-get will load 50% faster! from we_get.core.shell import Shell self.shell = Shell() - self.shell.item_color.update(self.config['item_color']) + if self.config['item_color']: + self.shell.item_color.update(self.config['item_color']) self.shell.shell(self.items, self.pargs) From eb0dc1e97ea18dc57e23c2a1cc1ccb9af0f1c838 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Thu, 4 Jun 2020 08:07:32 +0800 Subject: [PATCH 11/23] chg: dev: use None as default color set --- we_get/core/shell.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/we_get/core/shell.py b/we_get/core/shell.py index d3cc685..6cae738 100644 --- a/we_get/core/shell.py +++ b/we_get/core/shell.py @@ -15,7 +15,7 @@ from we_get.core.commands import COMMANDS from we_get.core.completer import WGCompleter from we_get.core.style import we_get_prompt_style -from we_get.core.utils import color, msg_error, msg_item, printc, printc_raw, ITEM_COLOR_SET +from we_get.core.utils import color, msg_error, msg_item, printc, printc_raw PROMPT_TOOLKIT_V2 = prompt_toolkit.__version__.split('.')[0] == '2' if PROMPT_TOOLKIT_V2: @@ -31,7 +31,7 @@ def __init__(self): self.pargs = None self.items = None self.show_links = False - self.item_color = ITEM_COLOR_SET.copy() + self.item_color = None def prompt_usage(self): printc("white", "Usage: help") From c23ae718df15f3794ba22d74cbb7bec8439ab691 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Thu, 4 Jun 2020 08:32:30 +0800 Subject: [PATCH 12/23] chg: doc: use the template for readme --- README.rst | 4 ++- tools/README.rst.tmpl | 60 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 050d69a..50749df 100644 --- a/README.rst +++ b/README.rst @@ -66,13 +66,14 @@ Options ===================== ===================================================== -s --search= Search for a torrent. -l --list List top torrents from modules. --t --target= Select module to use or 'all' [default: all]. +-t --target= Select module to use or 'all' [default: all]. -L --links Output results as links. -J --json Output results in JSON format. -G --get-list List targets (supported web-sites). -f --filter= Match text or regular expression in the torrent name. -n --results= Number of results to retrieve. -S --sort-type= Sort torrents by name/seeds [default: seeds]. +-c --config= Load config file. ===================== ===================================================== Video options @@ -83,6 +84,7 @@ Video options -g --genre= Try to select video genre for the torrent (action, comedy, etc..). ================ ================================================================== + See also ``we-get --help``. Python Module diff --git a/tools/README.rst.tmpl b/tools/README.rst.tmpl index 3300417..8764c42 100644 --- a/tools/README.rst.tmpl +++ b/tools/README.rst.tmpl @@ -1,11 +1,16 @@ we-get: command-line tool for searching torrents. ################################################# -.. image:: https://img.shields.io/github/license/mashape/apistatus.svg?style=flat-square :target: +.. image:: https://img.shields.io/github/license/mashape/apistatus.svg?style=flat-square + :target: https://github.com/rachmadaniHaryono/we-get/blob/master/LICENSE + + +.. image:: https://img.shields.io/aur/version/we-get-git.svg?maxAge=600 + :target: https://aur.archlinux.org/packages/we-get-git/ .. class:: head - .. image:: https://raw.githubusercontent.com/wiki/0xl3vi/we-get/screenshots/1.png + .. image:: https://raw.githubusercontent.com/rachmadaniHaryono/we-get/master/res/screenshot.png :alt: Main screenshot. :width: 100% :align: center @@ -117,7 +122,6 @@ Basic Usage text = subprocess.check_output("we-get --help", shell=True) - section_keyword = ['Options', 'Video options', 'General options'] section_keywords = ('Options', 'Video options', 'General options') result_dict = get_raw_text(text, section_keywords) result_dict = process_raw_text(result_dict) @@ -128,9 +132,46 @@ Basic Usage cog_obj.prout(output_text) ]]] [[[end]]] +See also ``we-get --help``. +Python Module +------------- + +.. code-block:: python + + >>> from we_get.core.we_get import WG + >>> we_get = WG() + >>> we_get.parse_arguments(['--search', 'ubuntu', '--target', 'all']) + >>> res = we_get.start(api_mode=True) + OrderedDict([ + ( + 'Ubuntu.MATE.16.04.2.[MATE][armhf][img.xz][Uzerus]', { + 'seeds': '260', + 'leeches': '2', + 'link': + 'magnet:?xt=urn:btih:D0F23C109D8662A3FE9338F75839AF8D57E5D4A9' + '&dn=Ubuntu+MATE+16.04.2+%5BMATE%5D%5Barmhf%5D%5Bimg.xz%5D%5BUzerus%5D' + '&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A80%2Fannounce' + '&tr=udp%3A%2F%2Ftracker.zer0day.to%3A1337%2Fannounce' + '&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969%2Fannounce' + '&tr=udp%3A%2F%2Fcoppersurfer.tk%3A6969%2Fannounce', + 'target': '1337x'} + ), + ... + ]) + +Older version can use `sys.argv` to input the arguments + +.. code-block:: python + + >>> import sys + >>> from we_get.core.we_get import WG + >>> we_get = WG() + >>> sys.argv[1:] = ['--search', 'ubuntu', '--target', 'all'] + >>> we_get.parse_arguments() + >>> we_get.start(api_mode=True) + ... -See also ``we-get --help``. Supported websites ------------------ @@ -154,3 +195,14 @@ Licence ------- MIT: `LICENSE `_. + +Testing +======= + +Dependencies +------------ + +* pytest +* pytest-flake8 + +Run tests with ``python -m pytest --flake8``. From 7af33a1b00e89a60a9ff44ac0642e8bff4519f10 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Sat, 5 Sep 2020 22:23:51 +0800 Subject: [PATCH 13/23] fix: dev: no config --- we_get/core/we_get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/we_get/core/we_get.py b/we_get/core/we_get.py index 5049ba5..bd99e3f 100644 --- a/we_get/core/we_get.py +++ b/we_get/core/we_get.py @@ -193,7 +193,7 @@ def run(self, api_mode=False): # we-get will load 50% faster! from we_get.core.shell import Shell self.shell = Shell() - if self.config['item_color']: + if hasattr(self, 'config') and self.config['item_color']: self.shell.item_color.update(self.config['item_color']) self.shell.shell(self.items, self.pargs) From 0fd7d5e19970df72dbb071e775c655f67b07dd62 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Sat, 5 Sep 2020 22:28:04 +0800 Subject: [PATCH 14/23] fix: dev: return empty when JSONDecodeError --- we_get/modules/yts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/we_get/modules/yts.py b/we_get/modules/yts.py index b572fe3..21e8c53 100644 --- a/we_get/modules/yts.py +++ b/we_get/modules/yts.py @@ -60,7 +60,10 @@ def search(self): def list(self): url = "%s%s" % (BASE_URL, LIST_LOC % (self.quality, self.genre)) - data = json.loads(self.module.http_get_request(url)) + try: + data = json.loads(self.module.http_get_request(url)) + except json.decoder.JSONDecodeError: + return self.items try: api = data['data']['movies'] except KeyError: From efeb436501443b94460da0676d0255ab1059a211 Mon Sep 17 00:00:00 2001 From: philhabell Date: Sun, 27 Sep 2020 23:01:10 +0100 Subject: [PATCH 15/23] Fixing the pirate bay module - Fixing url for tpb and update string concatination to f strings - Switching out regex searching for BeautifulSoup - Adding Beautiful Soup to setup install requirments - Adding new Beautiful Soup to READMEs --- README.rst | 2 +- setup.py | 1 + tools/README.rst.tmpl | 2 +- we_get/modules/the_pirate_bay.py | 37 +++++++++++++++----------------- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index 050d69a..08cc928 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ or with ``pip`` Dependencies ============ -* `prompt_toolkit `_, `docopt `_, `colorama `_ +* `prompt_toolkit `_, `docopt `_, `colorama `_, `beautifulsoup4 `_ and `Python `_ 3.5 or above diff --git a/setup.py b/setup.py index 1a64e2f..40adcab 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ install_requires=[ 'colorama', 'docopt', + 'beautifulsoup4', 'prompt_toolkit>=2.0.3', 'Pygments>=2.2.0', ], diff --git a/tools/README.rst.tmpl b/tools/README.rst.tmpl index 3300417..b3554bd 100644 --- a/tools/README.rst.tmpl +++ b/tools/README.rst.tmpl @@ -35,7 +35,7 @@ or with ``pip`` Dependencies ============ -* `prompt_toolkit `_, `docopt `_, `colorama `_ +* `prompt_toolkit `_, `docopt `_, `colorama `_, `beautifulsoup4 `_ and `Python `_ 3.5 or above diff --git a/we_get/modules/the_pirate_bay.py b/we_get/modules/the_pirate_bay.py index f3708e7..81857a0 100644 --- a/we_get/modules/the_pirate_bay.py +++ b/we_get/modules/the_pirate_bay.py @@ -2,12 +2,13 @@ Copyright (c) 2016-2019 we-get developers (https://github.com/rachmadaniHaryono/we-get/) See the file 'LICENSE' for copying permission """ +from bs4 import BeautifulSoup from we_get.core.module import Module import re BASE_URL = "https://www1.thepiratebay3.to" -SEARCH_LOC = "/search/%s/1/7/0" +SEARCH_LOC = "/s/?q=" LIST_LOC = "/top/all" @@ -33,38 +34,34 @@ def parse_pargs(self): self.action = "list" def _parse_data(self, data): - data = data.replace('\t', '').replace('\n', '') - items = re.findall( - r'
(.*?)
', data) + soup = BeautifulSoup(data, 'html.parser') + items = soup.find_all("tr") seeds = None leeches = None magnet = None for item in items: - seeds, leeches = re.findall( - r'(\d+)', item - ) - magnet = re.findall(r'href=[\'"]?([^\'">]+)', item)[1] - user_status = re.findall(r' 1: + name = cols[1].a.contents[0] + magnet = cols[3].a['href'] + seeds = cols[5].contents[0] + leeches = cols[6].contents[0] user_status = None - name = self.module.fix_name(self.module.magnet2name(magnet)) - self.items.update({ - name: { - 'seeds': seeds, 'leeches': leeches, - 'link': magnet, 'user_status': user_status} - }) + self.items.update({ + name: { + 'seeds': seeds, 'leeches': leeches, + 'link': magnet, 'user_status': user_status} + }) def search(self): - url = "%s%s" % (BASE_URL, SEARCH_LOC % (self.search_query)) + url = f"{BASE_URL}{SEARCH_LOC}{self.search_query}" data = self.module.http_get_request(url) self._parse_data(data) return self.items def list(self): - url = "%s%s" % (BASE_URL, LIST_LOC) + url = f"{BASE_URL}{LIST_LOC}" data = self.module.http_get_request(url) self._parse_data(data) return self.items From 6bae3869da70669a3e8fad2729e47316ba2d003a Mon Sep 17 00:00:00 2001 From: philhabell Date: Mon, 26 Oct 2020 00:28:11 +0000 Subject: [PATCH 16/23] Adding sfw filter to tph module and updating readme --- README.rst | 3 ++- we_get/modules/the_pirate_bay.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 6cb11a0..1ba856b 100644 --- a/README.rst +++ b/README.rst @@ -73,7 +73,8 @@ Options -f --filter= Match text or regular expression in the torrent name. -n --results= Number of results to retrieve. -S --sort-type= Sort torrents by name/seeds [default: seeds]. --c --config= Load config file. +-c --config= Load config file + --sfw restrict results to safe for work content (the_pirate_bay only) ===================== ===================================================== Video options diff --git a/we_get/modules/the_pirate_bay.py b/we_get/modules/the_pirate_bay.py index cea7471..bebcc20 100644 --- a/we_get/modules/the_pirate_bay.py +++ b/we_get/modules/the_pirate_bay.py @@ -10,7 +10,7 @@ BASE_URL = "https://www1.thepiratebay3.to" SEARCH_LOC = "/s/?q=" LIST_LOC = "/top/all" - +SFW_FILTER = "&audio=on&video=on&apps=on&games=on&other=on&category=0" class the_pirate_bay(object): """ the_pirate_bay module for we-get. @@ -24,6 +24,7 @@ def __init__(self, pargs): self.module = Module() self.parse_pargs() self.items = dict() + self.filter = "" def parse_pargs(self): for opt in self.pargs: @@ -32,6 +33,8 @@ def parse_pargs(self): self.search_query = self.pargs[opt][0].replace(' ', '-') elif opt == "--list": self.action = "list" + if opt == "--sfw": + self.filter = SFW_FILTER def _parse_data(self, data): soup = BeautifulSoup(data, 'html.parser') @@ -55,7 +58,7 @@ def _parse_data(self, data): }) def search(self): - url = f"{BASE_URL}{SEARCH_LOC}{self.search_query}" + url = f"{BASE_URL}{SEARCH_LOC}{self.search_query}{self.filter}" data = self.module.http_get_request(url) self._parse_data(data) return self.items From 3ab4fe8bf1ec910a05b51a095a18e7e1ead29cbc Mon Sep 17 00:00:00 2001 From: philhabell Date: Mon, 26 Oct 2020 00:48:38 +0000 Subject: [PATCH 17/23] Adding sfw arg to __doc__ and updating readme again --- README.rst | 2 +- we_get/core/we_get.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1ba856b..e16a026 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,7 @@ Options -n --results= Number of results to retrieve. -S --sort-type= Sort torrents by name/seeds [default: seeds]. -c --config= Load config file - --sfw restrict results to safe for work content (the_pirate_bay only) +-w --sfw Restrict results to safe for work content (the_pirate_bay only) ===================== ===================================================== Video options diff --git a/we_get/core/we_get.py b/we_get/core/we_get.py index bd99e3f..a02f7f3 100644 --- a/we_get/core/we_get.py +++ b/we_get/core/we_get.py @@ -38,6 +38,7 @@ -n --results= Number of results to retrieve. -S --sort-type= Sort torrents by name/seeds [default: seeds]. -c --config= Load config file. + -w --sfw Restrict results to safe for work content (the_pirate_bay only) Video options: -q --quality= Try to match quality for the torrent (720p,1080p, ...). From 9c0de346f240dc81ef1cc7e9585879a141570eb0 Mon Sep 17 00:00:00 2001 From: philhabell Date: Mon, 26 Oct 2020 01:10:03 +0000 Subject: [PATCH 18/23] moving filter var creation before parse_pargs to fix it --- we_get/modules/the_pirate_bay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/we_get/modules/the_pirate_bay.py b/we_get/modules/the_pirate_bay.py index bebcc20..90fc66a 100644 --- a/we_get/modules/the_pirate_bay.py +++ b/we_get/modules/the_pirate_bay.py @@ -21,10 +21,10 @@ def __init__(self, pargs): self.pargs = pargs self.action = None self.search_query = None + self.filter = "" self.module = Module() self.parse_pargs() self.items = dict() - self.filter = "" def parse_pargs(self): for opt in self.pargs: From 6c71fe216e478c423af4819d2c391b9fde594b5f Mon Sep 17 00:00:00 2001 From: philhabell Date: Tue, 17 Nov 2020 23:29:39 +0000 Subject: [PATCH 19/23] Migrating to tpb api: - migrating tph module to use the api instead of scraping from the frontend - fixing user status bug in utils --- we_get/core/utils.py | 3 +- we_get/modules/the_pirate_bay.py | 58 ++++++++++++++++---------------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/we_get/core/utils.py b/we_get/core/utils.py index 3887616..7497590 100644 --- a/we_get/core/utils.py +++ b/we_get/core/utils.py @@ -142,13 +142,12 @@ def msg_item( seeds = items['seeds'] target = items['target'] user_status = items.get('user_status', None) + user_status_text = '' if user_status and user_status is not None: if user_status == 'vip': user_status_text = color(cset['user_status_vip'], user_status) else: color(cset['user_status'], user_status) - else: - user_status_text = '' text = ( "%s %s [%s/%s] %s" % ( diff --git a/we_get/modules/the_pirate_bay.py b/we_get/modules/the_pirate_bay.py index 90fc66a..20d025d 100644 --- a/we_get/modules/the_pirate_bay.py +++ b/we_get/modules/the_pirate_bay.py @@ -6,15 +6,19 @@ from we_get.core.module import Module import re +import urllib +import json + + +API_URL = "https://apibay.org" +API_SEARCH_LOC = "/q.php?q=" +ALI_LIST_LOC = "/precompiled/data_top100_all.json" +API_SFW_FILTER = "&cat=100,200,300,400,600" +API_TRACKERS = "&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2850%2Fannounce&tr=udp%3A%2F%2F9.rarbg.to%3A2920%2Fannounce&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969%2Fannounce" -BASE_URL = "https://www1.thepiratebay3.to" -SEARCH_LOC = "/s/?q=" -LIST_LOC = "/top/all" -SFW_FILTER = "&audio=on&video=on&apps=on&games=on&other=on&category=0" class the_pirate_bay(object): - """ the_pirate_bay module for we-get. - """ + """the_pirate_bay module for we-get.""" def __init__(self, pargs): self.links = None @@ -30,41 +34,37 @@ def parse_pargs(self): for opt in self.pargs: if opt == "--search": self.action = "search" - self.search_query = self.pargs[opt][0].replace(' ', '-') + self.search_query = self.pargs[opt][0].replace(" ", "-") elif opt == "--list": self.action = "list" if opt == "--sfw": - self.filter = SFW_FILTER + self.filter = API_SFW_FILTER - def _parse_data(self, data): - soup = BeautifulSoup(data, 'html.parser') - items = soup.find_all("tr") - seeds = None - leeches = None - magnet = None + def generate_magnet(self, data): + return f"magnet:?xt=urn:btih:{data['info_hash']}&dn={urllib.parse.quote(data['name'])}{API_TRACKERS}" - for item in items: - cols = item.find_all("td") - if len(cols) > 1: - name = cols[1].a.contents[0] - magnet = cols[3].a['href'] - seeds = cols[5].contents[0] - leeches = cols[6].contents[0] - user_status = None - self.items.update({ - name: { - 'seeds': seeds, 'leeches': leeches, - 'link': magnet, 'user_status': user_status} - }) + def _parse_data(self, data): + for row in json.loads(data): + print(row) + self.items.update( + { + row["name"]: { + "seeds": row["seeders"], + "leeches": row["leechers"], + "link": self.generate_magnet(row), + "user_status": row["status"], + } + } + ) def search(self): - url = f"{BASE_URL}{SEARCH_LOC}{self.search_query}{self.filter}" + url = f"{API_URL}{API_SEARCH_LOC}{self.search_query}{self.filter}" data = self.module.http_get_request(url) self._parse_data(data) return self.items def list(self): - url = f"{BASE_URL}{LIST_LOC}" + url = f"{API_URL}{ALI_LIST_LOC}" data = self.module.http_get_request(url) self._parse_data(data) return self.items From 47f6ec7a9843973162ee08910a06508193278318 Mon Sep 17 00:00:00 2001 From: philhabell Date: Tue, 17 Nov 2020 23:37:02 +0000 Subject: [PATCH 20/23] removing print --- we_get/modules/the_pirate_bay.py | 1 - 1 file changed, 1 deletion(-) diff --git a/we_get/modules/the_pirate_bay.py b/we_get/modules/the_pirate_bay.py index 20d025d..a807a24 100644 --- a/we_get/modules/the_pirate_bay.py +++ b/we_get/modules/the_pirate_bay.py @@ -45,7 +45,6 @@ def generate_magnet(self, data): def _parse_data(self, data): for row in json.loads(data): - print(row) self.items.update( { row["name"]: { From f1a75dfe0494446163d7386ed221f3f1607769a0 Mon Sep 17 00:00:00 2001 From: philhabell Date: Thu, 19 Nov 2020 21:42:53 +0000 Subject: [PATCH 21/23] Fixing tests and adding vcrpy to reqs --- README.rst | 2 +- requirements-dev.txt | 1 + tests/test_arguments.py | 2 +- tests/test_core_we_get.py | 4 +++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index e16a026..56222a0 100644 --- a/README.rst +++ b/README.rst @@ -73,7 +73,7 @@ Options -f --filter= Match text or regular expression in the torrent name. -n --results= Number of results to retrieve. -S --sort-type= Sort torrents by name/seeds [default: seeds]. --c --config= Load config file +-c --config= Load config file. -w --sfw Restrict results to safe for work content (the_pirate_bay only) ===================== ===================================================== diff --git a/requirements-dev.txt b/requirements-dev.txt index 7517571..4cc182b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1 +1,2 @@ cogapp==2.5.1 +vcrpy==4.1.1 diff --git a/tests/test_arguments.py b/tests/test_arguments.py index c2e58af..9ba9e32 100755 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -11,7 +11,7 @@ class TestsArguments(unittest.TestCase): def test_number_of_arguments(self): args = docopt.docopt(__doc__) - self.assertEqual(len(args), 13) + self.assertEqual(len(args), 15) def test_required_argument_search(self): sys.argv = ['prog_name', '--search'] diff --git a/tests/test_core_we_get.py b/tests/test_core_we_get.py index 5d30afc..7eb67fb 100644 --- a/tests/test_core_we_get.py +++ b/tests/test_core_we_get.py @@ -11,6 +11,7 @@ [None, {'arguments': None, 'parguments': {}, 'we_get_run': 0}], [['--search', 'ubuntu'], { 'arguments': { + '--config': [], '--filter': [], '--genre': [], '--get-list': 0, @@ -21,6 +22,7 @@ '--quality': [], '--results': [], '--search': ['ubuntu'], + '--sfw': 0, '--sort-type': [], '--target': ['all'], '--version': 0 @@ -52,7 +54,7 @@ def test_parse_arguments(argv, exp_res): { '--filter': [], '--genre': [], '--get-list': 0, '--help': 0, '--json': 0, '--links': 0, '--list': 0, '--quality': [], '--results': [], '--search': [], - '--sort-type': [], '--target': ['all'], '--version': 0} + '--sort-type': [], '--target': ['all'], '--version': 0, '--config': [], '--sfw': 0} ], ], ) From ab6a2ea17453d2a6259b9d55b065358729e8294b Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Fri, 20 Nov 2020 06:44:08 +0800 Subject: [PATCH 22/23] chg: test: pby vcr use new_episodes --- tests/test_the_pirate_bay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_the_pirate_bay.py b/tests/test_the_pirate_bay.py index e29bc63..5659b99 100644 --- a/tests/test_the_pirate_bay.py +++ b/tests/test_the_pirate_bay.py @@ -7,7 +7,7 @@ def test_search(): from we_get.modules.the_pirate_bay import the_pirate_bay cl = the_pirate_bay('') cl.search_query = 'ubuntu' - with vcr.use_cassette('fixtures/test_the_pirate_bay_test_search.yaml'): + with vcr.use_cassette('fixtures/test_the_pirate_bay_test_search.yaml', record_mode='new_episodes'): res = cl.search() assert res assert any('user_status' in res[k] for k in res) From 4f573890cd9112bcc1651782178d6fcfe962fc62 Mon Sep 17 00:00:00 2001 From: rachmadaniHaryono Date: Fri, 20 Nov 2020 06:47:06 +0800 Subject: [PATCH 23/23] new: doc: version 1.1.1 --- changelog.rst | 5 +++-- setup.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/changelog.rst b/changelog.rst index 9f65658..543af1c 100644 --- a/changelog.rst +++ b/changelog.rst @@ -5,14 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on `Keep a Changelog`_ and this project adheres to `Semantic Versioning`_. -[Unreleased] ------------- +1.1.1 - 2020-11-20 +------------------ New ~~~ - user status for pirate bay - argv for arguments +- pby use api Changes ~~~~~~~ diff --git a/setup.py b/setup.py index f27ff3a..d764ea4 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup, find_packages -version = '1.1.0' +version = '1.1.1' setup( name='we-get',