[gelbooru] improve pagination logic for meta tags (#5478)

similar to 494acab
mikf · Apr 15, 2024 · 257e9fb · 257e9fb
1 parent 4cf24f1
commit 257e9fb
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 5 deletions.
diff --git a/gallery_dl/extractor/gelbooru.py b/gallery_dl/extractor/gelbooru.py
@@ -51,19 +51,44 @@ def _pagination(self, params):
         params["pid"] = self.page_start
         params["limit"] = self.per_page
         limit = self.per_page // 2
+        pid = False
+
+        if "tags" in params:
+            tags = params["tags"].split()
+            op = "<"
+            id = False
+
+            for tag in tags:
+                if tag.startswith("sort:"):
+                    if tag == "sort:id:asc":
+                        op = ">"
+                    elif tag == "sort:id" or tag.startswith("sort:id:"):
+                        op = "<"
+                    else:
+                        pid = True
+                elif tag.startswith("id:"):
+                    id = True
+
+            if not pid:
+                if id:
+                    tag = "id:" + op
+                    tags = [t for t in tags if not t.startswith(tag)]
+                tags = "{} id:{}".format(" ".join(tags), op)
 
         while True:
             posts = self._api_request(params)
 
-            for post in posts:
-                yield post
+            yield from posts
 
             if len(posts) < limit:
                 return
 
-            if "pid" in params:
-                del params["pid"]
-            params["tags"] = "{} id:<{}".format(self.tags, post["id"])
+            if pid:
+                params["pid"] += 1
+            else:
+                if "pid" in params:
+                    del params["pid"]
+                params["tags"] = tags + str(posts[-1]["id"])
 
     def _pagination_html(self, params):
         url = self.root + "/index.php"

diff --git a/test/results/gelbooru.py b/test/results/gelbooru.py
@@ -39,6 +39,22 @@
     "#sha1_url": "845a61aa1f90fb4ced841e8b7e62098be2e967bf",
 },
 
+{
+    "#url"     : "https://gelbooru.com/index.php?page=post&s=list&tags=id:>=67800+id:<=68000",
+    "#comment" : "meta tags (#5478)",
+    "#category": ("booru", "gelbooru", "tag"),
+    "#class"   : gelbooru.GelbooruTagExtractor,
+    "#count"   : 187,
+},
+
+{
+    "#url"     : "https://gelbooru.com/index.php?page=post&s=list&tags=id:>=67800+id:<=68000+sort:id:asc",
+    "#comment" : "meta + sort tags (#5478)",
+    "#category": ("booru", "gelbooru", "tag"),
+    "#class"   : gelbooru.GelbooruTagExtractor,
+    "#count"   : 187,
+},
+
 {
     "#url"     : "https://gelbooru.com/index.php?page=pool&s=show&id=761",
     "#category": ("booru", "gelbooru", "pool"),