diff --git a/docs/gallery-dl.conf b/docs/gallery-dl.conf index 2a7f8f203b..c7382f3e58 100644 --- a/docs/gallery-dl.conf +++ b/docs/gallery-dl.conf @@ -1,31 +1,83 @@ { + "#": "gallery-dl default configuration file", + + "#": "full documentation at", + "#": "https://gdl-org.github.io/docs/configuration.html", + "extractor": { + "#": "===============================================================", + "#": "==== General Extractor Options ==========================", + "#": "(these can be set as site-specific extractor options as well) ", + "base-directory": "./gallery-dl/", - "parent-directory": false, "postprocessors": null, - "archive": null, + "skip" : true, + "skip-filter" : null, + + "user-agent" : "auto", + "referer" : true, + "headers" : {}, + "ciphers" : null, + "tls12" : true, + "browser" : null, + "proxy" : null, + "proxy-env" : false, + "source-address": null, + "retries" : 4, + "retry-codes" : [], + "timeout" : 30.0, + "verify" : true, + "download" : true, + "fallback" : true, + + "archive" : null, + "archive-format": null, + "archive-prefix": null, + "archive-pragma": [], + "archive-event" : ["file"], + "archive-mode" : "file", + "cookies": null, + "cookies-select": null, "cookies-update": true, - "proxy": null, - "skip": true, - "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0", - "retries": 4, - "timeout": 30.0, - "verify": true, - "fallback": true, + "image-filter" : null, + "image-range" : null, + "image-unique" : false, + "chapter-filter": null, + "chapter-range" : null, + "chapter-unique": false, - "sleep": 0, - "sleep-request": 0, - "sleep-extractor": 0, + "keywords" : {}, + "keywords-eval" : false, + "keywords-default" : null, + + "parent-directory": false, + "parent-metadata" : false, + "parent-skip" : false, "path-restrict": "auto", - "path-replace": "_", - "path-remove": "\\u0000-\\u001f\\u007f", - "path-strip": "auto", + "path-replace" : "_", + "path-remove" : "\\u0000-\\u001f\\u007f", + "path-strip" : "auto", "path-extended": true, + "metadata-extractor": null, + "metadata-http" : null, + "metadata-parent" : null, + "metadata-path" : null, + "metadata-url" : null, + "metadata-version" : null, + + "sleep" : 0, + "sleep-request" : 0, + "sleep-extractor": 0, + "sleep-429" : 60.0, + + "actions": [], + "input" : null, + "netrc" : false, "extension-map": { "jpeg": "jpg", "jpe" : "jpg", @@ -35,157 +87,313 @@ }, + "#": "===============================================================", + "#": "==== Site-specific Extractor Options ====================", + + "ao3": + { + "username": "", + "password": "", + "sleep-request": "0.5-1.5", + + "formats": ["pdf"] + }, "artstation": { - "external": false, - "pro-first": true + "external" : false, + "max-posts": null, + "previews" : false, + "videos" : true, + + "search": { + "pro-first": true + } }, "aryion": { - "username": null, - "password": null, + "username": "", + "password": "", + "recursive": true }, - "bbc": { + "bbc": + { "width": 1920 }, - "blogger": + "behance": { - "videos": true + "sleep-request": "2.0-4.0", + + "modules": ["image", "video", "mediacollection", "embed"] }, - "cyberdrop": + "bilibili": { - "domain": null + "sleep-request": "2.0-4.0" }, - "danbooru": + "bluesky": { - "username": null, - "password": null, - "external": false, + "username": "", + "password": "", + + "include" : ["media"], + "metadata": false, + "quoted" : false, + "reposts" : false, + "videos" : true, + + "post": { + "depth": 0 + } + }, + "boosty": + { + "allowed" : true, + "bought" : false, "metadata": false, - "ugoira": false + "videos" : true + }, + "bunkr": + { + "tlds": false }, - "derpibooru": + "cien": + { + "sleep-request": "1.0-2.0", + "files": ["image", "video", "download", "gallery"] + }, + "civitai": { "api-key": null, - "filter": 56027 + "sleep-request": "0.5-1.5", + + "api" : "trpc", + "files" : ["image"], + "include" : ["user-models", "user-posts"], + "metadata": false, + "nsfw" : true, + "quality" : "original=true" + }, + "cohost": + { + "asks" : true, + "pinned" : false, + "replies": true, + "shares" : true + }, + "coomerparty": + { + "username": "", + "password": "", + + "announcements": false, + "comments" : false, + "dms" : false, + "duplicates" : false, + "favorites" : "artist", + "files" : ["attachments", "file", "inline"], + "max-posts" : null, + "metadata" : false, + "revisions" : false, + "order-revisions": "desc" + }, + "cyberdrop": + { + "domain": null }, "deviantart": { - "client-id": null, + "client-id" : null, "client-secret": null, "refresh-token": null, - "auto-watch": false, - "auto-unwatch": false, - "comments": false, - "extra": false, - "flat": true, - "folders": false, - "group": true, - "include": "gallery", - "journals": "html", - "jwt": false, - "mature": true, - "metadata": false, - "original": true, - "pagination": "api", - "public": true, - "quality": 100, - "wait-min": 0 - }, - "e621": - { - "username": null, - "password": null + "auto-watch" : false, + "auto-unwatch" : false, + "comments" : false, + "comments-avatars": false, + "extra" : false, + "flat" : true, + "folders" : false, + "group" : true, + "include" : "gallery", + "intermediary" : true, + "journals" : "html", + "jwt" : false, + "mature" : true, + "metadata" : false, + "original" : true, + "pagination" : "api", + "previews" : false, + "public" : true, + "quality" : 100, + "wait-min" : 0, + + "avatar": { + "formats": null + } }, "exhentai": { - "username": null, - "password": null, - "domain": "auto", - "limits": true, + "username": "", + "password": "", + "cookies" : null, + "sleep-request": "3.0-6.0", + + "domain" : "auto", + "fav" : null, + "gp" : "resized", + "limits" : null, "metadata": false, "original": true, - "sleep-request": 5.0 + "source" : null, + "tags" : false, + "fallback-retries": 2 + }, + "fanbox": + { + "cookies" : null, + + "comments": false, + "embeds" : true, + "metadata": false }, "flickr": { - "exif": false, + "access-token" : null, + "access-token-secret": null, + "sleep-request" : "1.0-2.0", + + "contexts": false, + "exif" : false, "metadata": false, "size-max": null, - "videos": true + "videos" : true }, "furaffinity": { + "cookies" : null, + "descriptions": "text", - "external": false, - "include": "gallery", - "layout": "auto" + "external" : false, + "include" : ["gallery"], + "layout" : "auto" }, "gelbooru": { "api-key": null, - "user-id": null + "user-id": null, + + "favorite": { + "order-posts": "desc" + } }, - "gofile": { + "generic": + { + "enabled": false + }, + "gofile": + { "api-token": null, - "website-token": null + "website-token": null, + "recursive": false }, "hentaifoundry": { - "include": "pictures" + "include": ["pictures"] }, "hitomi": { - "format": "webp", - "metadata": false + "format": "webp" }, "idolcomplex": { - "username": null, - "password": null, - "sleep-request": 5.0 + "username": "", + "password": "", + "referer" : false, + "sleep-request": "3.0-6.0" }, - "imagechest": { + "imagechest": + { "access-token": null }, + "imagefap": + { + "sleep-request": "2.0-4.0" + }, "imgbb": { - "username": null, - "password": null + "username": "", + "password": "" }, "imgur": { + "client-id": null, "mp4": true }, "inkbunny": { - "username": null, - "password": null, + "username": "", + "password": "", "orderby": "create_datetime" }, "instagram": { - "api": "rest", "cookies": null, - "include": "posts", + "sleep-request": "6.0-12.0", + + "api" : "rest", + "cursor" : true, + "include" : "posts", + "max-posts" : null, + "metadata" : false, "order-files": "asc", "order-posts": "asc", - "previews": false, - "sleep-request": [6.0, 12.0], + "previews" : false, + "videos" : true + }, + "itaku": + { + "sleep-request": "0.5-1.5", "videos": true }, + "kemonoparty": + { + "username": "", + "password": "", + + "announcements": false, + "comments" : false, + "dms" : false, + "duplicates" : false, + "favorites" : "artist", + "files" : ["attachments", "file", "inline"], + "max-posts" : null, + "metadata" : false, + "revisions" : false, + "order-revisions": "desc" + }, "khinsider": { "format": "mp3" }, + "koharu": + { + "username": "", + "password": "", + "sleep-request": "0.5-1.5", + + "cbz" : true, + "format": ["0", "1600", "1280", "980", "780"], + "tags" : false + }, "luscious": { "gif": false }, "mangadex": { + "username": "", + "password": "", + "api-server": "https://api.mangadex.org", "api-parameters": null, "lang": null, @@ -193,248 +401,562 @@ }, "mangoxo": { - "username": null, - "password": null - }, - "misskey": { - "access-token": null, - "renotes": false, - "replies": true + "username": "", + "password": "" }, "newgrounds": { - "username": null, - "password": null, - "flash": true, - "format": "original", - "include": "art" + "username": "", + "password": "", + "sleep-request": "0.5-1.5", + + "flash" : true, + "format" : "original", + "include": ["art"] }, - "nijie": + "nsfwalbum": { - "username": null, - "password": null, - "include": "illustration,doujin" - }, - "nitter": { - "quoted": false, - "retweets": false, - "videos": true + "referer": false }, "oauth": { "browser": true, - "cache": true, - "host": "localhost", - "port": 6414 + "cache" : true, + "host" : "localhost", + "port" : 6414 }, "paheal": { "metadata": false }, + "patreon": + { + "cookies": null, + + "files" : ["images", "image_large", "attachments", "postfile", "content"] + }, "pillowfort": { + "username": "", + "password": "", + "external": false, - "inline": true, - "reblogs": false + "inline" : true, + "reblogs" : false }, "pinterest": { - "domain": "auto", + "domain" : "auto", "sections": true, - "videos": true + "stories" : true, + "videos" : true + }, + "pixeldrain": + { + "api-key": null }, "pixiv": { "refresh-token": null, - "include": "artworks", - "embeds": false, - "metadata": false, + "cookies" : null, + + "captions" : false, + "comments" : false, + "include" : ["artworks"], + "max-posts": null, + "metadata" : false, "metadata-bookmark": false, - "tags": "japanese", - "ugoira": true + "sanity" : true, + "tags" : "japanese", + "ugoira" : true, + + "covers" : false, + "embeds" : false, + "full-series": false }, - "reactor": + "plurk": { - "gif": false, - "sleep-request": 5.0 + "sleep-request": "0.5-1.5", + "comments": false + }, + "poipiku": + { + "sleep-request": "0.5-1.5" + }, + "pornpics": + { + "sleep-request": "0.5-1.5" + }, + "readcomiconline": + { + "sleep-request": "3.0-6.0", + + "captcha": "stop", + "quality": "auto" }, "reddit": { - "client-id": null, - "user-agent": null, + "client-id" : null, + "user-agent" : null, "refresh-token": null, - "comments": 0, + + "comments" : 0, "morecomments": false, - "date-min": 0, - "date-max": 253402210800, - "date-format": "%Y-%m-%dT%H:%M:%S", - "id-min": null, - "id-max": null, - "recursion": 0, - "videos": true + "embeds" : true, + "date-min" : 0, + "date-max" : 253402210800, + "date-format" : "%Y-%m-%dT%H:%M:%S", + "id-min" : null, + "id-max" : null, + "previews" : true, + "recursion" : 0, + "videos" : true }, "redgifs": { "format": ["hd", "sd", "gif"] }, + "rule34xyz": + { + "format": ["10", "40", "41", "2"] + }, "sankaku": { - "username": null, - "password": null, - "refresh": false + "username": "", + "password": "", + + "id-format": "numeric", + "refresh" : false, + "tags" : false }, "sankakucomplex": { "embeds": false, "videos": true }, + "scrolller": + { + "username": "", + "password": "", + "sleep-request": "0.5-1.5" + }, "skeb": { - "article": false, - "filters": null, + "article" : false, "sent-requests": false, - "thumbnails": false + "thumbnails" : false, + + "search": { + "filters": null + } }, "smugmug": { + "access-token" : null, + "access-token-secret": null, + "videos": true }, + "soundgasm": + { + "sleep-request": "0.5-1.5" + }, + "steamgriddb": + { + "animated" : true, + "epilepsy" : true, + "humor" : true, + "dimensions": "all", + "file-types": "all", + "languages" : "all,", + "nsfw" : true, + "sort" : "score_desc", + "static" : true, + "styles" : "all", + "untagged" : true, + "download-fake-png": true + }, "seiga": { - "username": null, - "password": null + "username": "", + "password": "", + "cookies" : null }, "subscribestar": { - "username": null, - "password": null + "username": "", + "password": "" + }, + "tapas": + { + "username": "", + "password": "" }, "tsumino": { - "username": null, - "password": null + "username": "", + "password": "" }, "tumblr": { - "avatar": false, - "external": false, - "inline": true, - "posts": "all", - "offset": 0, - "original": true, - "reblogs": true + "access-token" : null, + "access-token-secret": null, + + "avatar" : false, + "date-min" : 0, + "date-max" : null, + "external" : false, + "inline" : true, + "offset" : 0, + "original" : true, + "pagination": "offset", + "posts" : "all", + "ratelimit" : "abort", + "reblogs" : true, + "fallback-delay" : 120.0, + "fallback-retries": 2 + }, + "tumblrgallery": + { + "referer": false }, "twitter": { - "username": null, - "password": null, - "cards": false, - "conversations": false, - "pinned": false, - "quoted": false, - "replies": true, - "retweets": false, - "strategy": null, - "text-tweets": false, - "twitpic": false, - "unique": true, - "users": "user", - "videos": true + "username" : "", + "username-alt": "", + "password" : "", + "cookies" : null, + + "ads" : false, + "cards" : false, + "cards-blacklist": [], + "csrf" : "cookies", + "cursor" : true, + "expand" : false, + "include" : ["timeline"], + "locked" : "abort", + "logout" : true, + "pinned" : false, + "quoted" : false, + "ratelimit" : "wait", + "relogin" : true, + "replies" : true, + "retweets" : false, + "size" : ["orig", "4096x4096", "large", "medium", "small"], + "text-tweets" : false, + "tweet-endpoint": "auto", + "transform" : true, + "twitpic" : false, + "unavailable" : false, + "unique" : true, + "users" : "user", + "videos" : true, + + "timeline": { + "strategy": "auto" + }, + "tweet": { + "conversations": false + } }, "unsplash": { "format": "raw" }, + "urlgalleries": + { + "sleep-request": "0.5-1.5" + }, + "vipergirls": + { + "username": "", + "password": "", + "sleep-request": "0.5", + + "domain" : "vipergirls.to", + "like" : false + }, + "vk": + { + "sleep-request": "0.5-1.5", + "offset": 0 + }, "vsco": { - "videos": true + "include": ["gallery"], + "videos" : true }, "wallhaven": { - "api-key": null, - "metadata": false, - "include": "uploads" + "api-key" : null, + "include" : ["uploads"], + "metadata": false }, "weasyl": { - "api-key": null, + "api-key" : null, "metadata": false }, "weibo": { + "sleep-request": "1.0-2.0", + + "gifs" : true, + "include" : ["feed"], "livephoto": true, - "retweets": true, - "videos": true + "retweets" : false, + "videos" : true }, "ytdl": { - "enabled": false, - "format": null, - "generic": true, - "logging": true, - "module": null, - "raw-options": null + "cmdline-args": null, + "config-file" : null, + "enabled" : false, + "format" : null, + "generic" : true, + "logging" : true, + "module" : null, + "raw-options" : null }, "zerochan": { - "username": null, - "password": null, - "metadata": false + "username": "", + "password": "", + "sleep-request": "0.5-1.5", + + "metadata" : false, + "pagination": "api", + "redirects" : false + }, + + + "#": "===============================================================", + "#": "==== Base-Extractor and Instance Options ================", + + "blogger": + { + "api-key": null, + "videos" : true + }, + + "Danbooru": + { + "sleep-request": "0.5-1.5", + + "external" : false, + "metadata" : false, + "threshold": "auto", + "ugoira" : false + }, + "danbooru": + { + "username": "", + "password": "" + }, + "atfbooru": + { + "username": "", + "password": "" + }, + "aibooru": + { + "username": "", + "password": "" + }, + "booruvar": + { + "username": "", + "password": "" + }, + + "E621": + { + "sleep-request": "0.5-1.5", + + "metadata" : false, + "threshold": "auto" + }, + "e621": + { + "username": "", + "password": "" + }, + "e926": + { + "username": "", + "password": "" + }, + "e6ai": + { + "username": "", + "password": "" + }, + + "foolfuuka": + { + "sleep-request": "0.5-1.5" + }, + "archivedmoe": + { + "referer": false + }, + + "mastodon": + { + "access-token": null, + "cards" : false, + "reblogs" : false, + "replies" : true, + "text-posts" : false + }, + + "misskey": + { + "access-token": null, + "renotes" : false, + "replies" : true }, + + "Nijie": + { + "sleep-request": "2.0-4.0", + "include" : ["illustration", "doujin"] + }, + "nijie": + { + "username": "", + "password": "" + }, + "horne": + { + "username": "", + "password": "" + }, + + "nitter": + { + "quoted" : false, + "retweets": false, + "videos" : true + }, + + "philomena": + { + "api-key": null, + "sleep-request": "0.5-1.5", + + "svg" : true, + "filter": 2 + }, + "derpibooru": { + "filter": 56027 + }, + "ponybooru": { + "filter": 3 + }, + "twibooru": { + "sleep-request": "6.0-6.1" + }, + + "postmill": + { + "save-link-post-body": false + }, + + "reactor": + { + "sleep-request": "3.0-6.0", + "gif": false + }, + + "wikimedia": + { + "sleep-request": "1.0-2.0", + "limit": 50 + }, + "booru": { - "tags": false, - "notes": false + "tags" : false, + "notes": false, + "url" : "file_url" } }, + + "#": "===================================================================", + "#": "==== Downloader Options =====================================", + "downloader": { - "filesize-min": null, - "filesize-max": null, - "mtime": true, - "part": true, + "filesize-min" : null, + "filesize-max" : null, + "mtime" : true, + "part" : true, "part-directory": null, - "progress": 3.0, - "rate": null, - "retries": 4, - "timeout": 30.0, - "verify": true, + "progress" : 3.0, + "proxy" : null, + "rate" : null, + "retries" : 4, + "timeout" : 30.0, + "verify" : true, "http": { "adjust-extensions": true, - "chunk-size": 32768, - "headers": null, - "validate": true + "chunk-size" : 32768, + "consume-content" : false, + "enabled" : true, + "headers" : null, + "retry-codes" : [404, 429, 430], + "validate" : true }, "ytdl": { - "format": null, + "cmdline-args" : null, + "config-file" : null, + "enabled" : true, + "format" : null, "forward-cookies": false, - "logging": true, - "module": null, - "outtmpl": null, - "raw-options": null + "logging" : true, + "module" : null, + "outtmpl" : null, + "raw-options" : null } }, + + "#": "===================================================================", + "#": "==== Output Options =========================================", + "output": { - "mode": "auto", - "progress": true, - "shorten": true, - "ansi": false, - "colors": { - "success": "1;32", - "skip" : "2" - }, - "skip": true, - "log": "[{name}][{levelname}] {message}", - "logfile": null, - "unsupportedfile": null - }, + "ansi" : true, + "fallback" : true, + "mode" : "auto", + "private" : false, + "progress" : true, + "shorten" : true, + "skip" : true, + + "stdin" : null, + "stdout" : null, + "stderr" : null, + + "log" : "[{name}][{levelname}] {message}", + "logfile" : null, + "errorfile": null, + "unsupportedfile": null, - "netrc": false + "colors" : + { + "success": "1;32", + "skip" : "2", + "debug" : "0;37", + "info" : "1;37", + "warning": "1;33", + "error" : "1;31" + } + } } diff --git a/scripts/docs_compare.py b/scripts/docs_compare.py new file mode 100755 index 0000000000..5d3707884c --- /dev/null +++ b/scripts/docs_compare.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""Find missing settings in docs/gallery.conf""" + +import json +import util +import sys +import re + +from gallery_dl import text, extractor + + +def read(fname): + path = util.path("docs", fname) + try: + with open(path) as fp: + return fp.read() + except Exception as exc: + sys.exit("Unable to read {} ({}: {})".format( + path, exc.__class__.__name__, exc)) + + +DOCS = read("configuration.rst") +CONF = json.loads(read("gallery-dl.conf")) +EXTRS = list(extractor._list_classes()) + + +def opts_general(type): + # general opts + opts = re.findall( + r"(?m)^{}\.\*\.([\w-]+)".format(type), + DOCS) + extr = CONF[type] + + return { + type + ".*." + opt + for opt in opts + if opt not in extr + } + + +def opts_category(type): + # site opts + opts = re.findall( + r"(?m)^{}\.(?!\*)([\w-]+)\.([\w-]+)(?:\.([\w-]+))?".format(type), + DOCS) + extr = CONF[type] + + result = set() + for category, sub, opt in opts: + if category[0] == "[": + category = category[1:-1] + category_opts = extr.get(category) + if not category_opts: + result.add(category + ".*") + continue + + if not opt: + opt = sub + sub = None + elif sub: + category_opts = category_opts.get(sub) or () + if opt not in category_opts: + if sub: + opt = sub + "." + opt + result.add(category + "." + opt) + return result + + +def userpass(): + block = text.extr(DOCS, "extractor.*.username", "extractor.*.") + extr = CONF["extractor"] + + result = set() + for category in text.extract_iter(block, "* ``", "``"): + opts = extr[category] + if "username" not in opts or "password" not in opts: + result.add(category) + return result + + +def sleeprequest(): + block = text.extr(DOCS, "extractor.*.sleep-request", "extractor.*.") + + sleep = {} + for line in block.splitlines(): + line = line.strip() + if not line: + continue + + if line[0] == "*": + value = line.strip('* `"') + if value == "0": + break + elif line[0] == "`": + cat, _, sub = line.strip("`,").partition(":") + sleep[cat.strip("[]")] = value + + result = {} + for extr in EXTRS: + value = sleep.get(extr.category) + if value: + category = extr.category + else: + value = sleep.get(extr.basecategory) + if value: + category = extr.basecategory + else: + continue + + min, _, max = value.partition("-") + tup = (float(min), float(max)) + if tup != extr.request_interval: + result[category] = extr.request_interval + return result + + +write = sys.stdout.write + +opts = set() +opts.update(opts_general("extractor")) +opts.update(opts_general("downloader")) +opts.update(opts_category("extractor")) +opts.update(opts_category("downloader")) +if opts: + write("Missing Options:\n") + for opt in sorted(opts): + write(" {}\n".format(opt)) + write("\n") + + +categories = userpass() +if categories: + write("Missing username & password:\n") + for cat in sorted(categories): + write(" {}\n".format(cat)) + write("\n") + + +categories = sleeprequest() +if categories: + write("Wrong sleep_request:\n") + for cat, value in sorted(categories.items()): + write(" {}: {}\n".format(cat, value)) + write("\n")