Skip to content

Commit

Permalink
start implementing #402
Browse files Browse the repository at this point in the history
  • Loading branch information
Nandaka committed Oct 5, 2018
1 parent 8049cd1 commit d56bf81
Show file tree
Hide file tree
Showing 7 changed files with 1,005 additions and 7 deletions.
22 changes: 22 additions & 0 deletions PixivBrowserFactory.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from PixivException import PixivException
import PixivModelWhiteCube
import PixivModel
from PixivModelFanbox import Fanbox, FanboxArtist

defaultCookieJar = None
defaultConfig = None
Expand Down Expand Up @@ -616,6 +617,27 @@ def handleDebugTagSearchPage(self, response, url):
if self._config.debugHttp:
PixivHelper.safePrint(u"reply: {0}".format(PixivHelper.toUnicode(response)))

def fanboxGetSupportedUsers(self):
''' get all supported users from the list from https://www.pixiv.net/ajax/fanbox/support'''
url = 'https://www.pixiv.net/ajax/fanbox/support'
PixivHelper.print_and_log('info', 'Getting supported artists from ' + url)
# read the json response
response = self.open(url).read()
result = Fanbox(response)
return result

def fanboxGetPostsFromArtist(self, artist_id, next_url=""):
''' get all posts from the supported user from https://www.pixiv.net/ajax/fanbox/creator?userId=15521131 '''
if next_url == "":
url = "https://www.pixiv.net/ajax/fanbox/creator?userId={0}".format(artist_id)
else:
url = "https://www.pixiv.net" + next_url

PixivHelper.print_and_log('info', 'Getting posts from ' + url)
response = self.open(url).read()
result = FanboxArtist(artist_id, response)
return result


def getBrowser(config=None, cookieJar=None):
global defaultCookieJar
Expand Down
106 changes: 106 additions & 0 deletions PixivModelFanbox.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
import PixivException
import demjson


class Fanbox:
supportedArtist = None

def __init__(self, page):
js = demjson.decode(page)

if js["error"]:
raise PixivException("Error when requesting Fanbox", 9999, page)

if js["body"] is not None:
self.parseSupportedArtists(js["body"])

def parseSupportedArtists(self, js_body):
self.supportedArtist = list()
for creator in js_body:
self.supportedArtist.append(int(creator["user"]["userId"]))


class FanboxArtist:
artist_id = 0
posts = None
nextUrl = None
hasNextPage = False

def __init__(self, artist_id, page):
self.artist_id = int(artist_id)
js = demjson.decode(page)

if js["error"]:
raise PixivException("Error when requesting Fanbox artist: {0}".format(artist_id), 9999, page)

if js["body"] is not None:
self.parsePosts(js["body"])

def parsePosts(self, js_body):
self.posts = list()
for jsPost in js_body["post"]["items"]:
post_id = int(jsPost["id"])
post = FanboxPost(post_id, self, jsPost)
self.posts.append(post)

self.nextUrl = js_body["post"]["nextUrl"]
if self.nextUrl is not None and len(self.nextUrl) > 0:
self.hasNextPage = True


class FanboxPost:
post_id = 0
title = ""
coverImageUrl = ""
publishedDatetime = ""
updatedDatetime = ""
# image|text
type = ""
body_text = ""
images = None
likeCount = 0
parent = None
is_restricted = False

def __init__(self, post_id, parent, page):
self.images = list()
self.post_id = int(post_id)
self.parent = parent
self.parsePost(page)
if not self.is_restricted:
self.parseBody(page)
if self.type == 'image':
self.parseImages(page)

def parsePost(self, jsPost):
self.title = jsPost["title"]
self.coverImageUrl = jsPost["coverImageUrl"]
self.publishedDatetime = jsPost["publishedDatetime"]
self.updatedDatetime = jsPost["updatedDatetime"]
self.type = jsPost["type"]
self.likeCount = int(jsPost["likeCount"])
if jsPost["body"] is None:
self.is_restricted = True

def parseBody(self, jsPost):
self.body_text = jsPost["body"]["text"]

def parseImages(self, jsPost):
for image in jsPost["body"]["images"]:
self.images.append(image["originalUrl"])


class FanboxHelper:
def makeFilename(filename_format, url, artist, post, type, image_pos=0):
fileUrl = os.path.basename(url)
splittedUrl = fileUrl.split('.')
imageExtension = splittedUrl[1]
imageExtension = imageExtension.split('?')[0]

# fake it for now
if type != "cover":
filename = "{0}/FANBOX {1} {4} {2}.{3}".format(artist.artist_id, post.post_id, splittedUrl[0], imageExtension, image_pos)
else:
filename = "{0}/FANBOX {1} cover {2}.{3}".format(artist.artist_id, post.post_id, splittedUrl[0], imageExtension)

return filename
110 changes: 103 additions & 7 deletions PixivUtil2.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ def readinto_patch(self, b):
from PixivModel import PixivNewIllustBookmark, PixivGroup
from PixivException import PixivException
import PixivBrowserFactory
from PixivModelFanbox import FanboxHelper

from optparse import OptionParser

Expand Down Expand Up @@ -619,14 +620,14 @@ def process_image(artist=None, image_id=None, user_dir='', bookmark=False, searc
print('Processing Image Id:', image_id)

# check if already downloaded. images won't be downloaded twice - needed in process_image to catch any download
r = __dbManager__.selectImageByImageId(image_id,cols='save_name')
exists=False
in_db=False
r = __dbManager__.selectImageByImageId(image_id, cols='save_name')
exists = False
in_db = False
if r is not None:
exists=True
in_db=True
exists = True
in_db = True
if r is not None and __config__.alwaysCheckFileExists:
exists=__dbManager__.cleanupFileExists(r[0])
exists = __dbManager__.cleanupFileExists(r[0])

if r is not None and not __config__.alwaysCheckFileSize and exists:
if not __config__.overwrite and exists:
Expand Down Expand Up @@ -841,7 +842,7 @@ def process_image(artist=None, image_id=None, user_dir='', bookmark=False, searc
PixivHelper.writeUrlInDescription(image, __config__.urlBlacklistRegex, __config__.urlDumpFilename)

if in_db and not exists:
result = PixivConstant.PIXIVUTIL_CHECK_DOWNLOAD #There was something in the database which had not been downloaded
result = PixivConstant.PIXIVUTIL_CHECK_DOWNLOAD # There was something in the database which had not been downloaded

# Only save to db if all images is downloaded completely
if result == PixivConstant.PIXIVUTIL_OK or result == PixivConstant.PIXIVUTIL_SKIP_DUPLICATE or result == PixivConstant.PIXIVUTIL_SKIP_LOCAL_LARGER:
Expand Down Expand Up @@ -1401,6 +1402,9 @@ def menu():
print('11. Download Member Bookmark (/bookmark.php?id=)')
print('12. Download by Group Id')
print('------------------------')
print('f1. Download from supported artists (FANBOX)')
print('f2. Download by artist id (FANBOX)')
print('------------------------')
print('d. Manage database')
print('e. Export online bookmark')
print('m. Export online user bookmark')
Expand Down Expand Up @@ -1766,6 +1770,92 @@ def menu_export_online_user_bookmark(opisvalid, args):
export_bookmark(filename, 'n', 1, 0, member_id)


def menu_fanbox_download_supported_artist():
end_page = raw_input("Max Page = ") or 0

result = __br__.fanboxGetSupportedUsers()
if len(result.supportedArtist) == 0:
PixivHelper.print_and_log(info, "No supported artist!")
return
PixivHelper.print_and_log(info, "Found {0} supported artist(s)".format(len(result.supportedArtist)))
PixivHelper.print_and_log(info, result.supportedArtist)

for artist_id in result.supportedArtist:
processFanboxArtist(artist_id, end_page)


def processFanboxArtist(artist_id, end_page):
current_page = 1
while(True):
PixivHelper.print_and_log(info, "Processing {0}, page {1}".format(artist_id, current_page))
result_artist = __br__.fanboxGetPostsFromArtist(artist_id)

for post in result_artist.posts:
# cover image
if post.coverImageUrl is not None:
filename = FanboxHelper.makeFilename(__config__.filenameFormat,
post.coverImageUrl,
result_artist,
post,
"cover")
filename = PixivHelper.sanitizeFilename(filename, __config__.rootDirectory)
referer = "https://www.pixiv.net/fanbox/creator/{0}/post/{1}".format(artist_id, post.post_id)
(result, filename) = download_image(post.coverImageUrl,
filename,
referer,
__config__.overwrite,
__config__.retry,
__config__.backupOldFile,
post.post_id,
post)
else:
PixivHelper.print_and_log(info, "No Cover Image for post: {0}.".format(post.post_id))

# images
if post.type == 'images':
processFanboxImages(post)

if not result_artist.hasNextPage:
PixivHelper.print_and_log(info, "No more post for {0}".format(artist_id))
break
current_page = current_page + 1
if current_page > end_page:
PixivHelper.print_and_log(info, "Reaching page limit for {0}, limit {1}".format(artist_id, end_page))
break


def processFanboxImages(post):
if post.is_restricted:
PixivHelper.print_and_log(info, "Skipping post: {0} due to restricted post.".format(post.post_id))
return

current_page = 0
for image_url in post.images:
filename = FanboxHelper.makeFilename(__config__.filenameFormat,
image_url,
result_artist,
post,
"image",
curent_page)
filename = PixivHelper.sanitizeFilename(filename, __config__.rootDirectory)
referer = "https://www.pixiv.net/fanbox/creator/{0}/post/{1}".format(artist_id, post.post_id)
(result, filename) = download_image(image_url,
filename,
referer,
__config__.overwrite,
__config__.retry,
__config__.backupOldFile,
post.post_id,
post)
current_page = current_page + 1


def menu_fanbox_download_by_artist_id():
artist_id = raw_input("Artist ID = ")
end_page = raw_input("Max Page = ") or 0
processFanboxArtist(artist_id, end_page)


def menu_reload_config():
__log__.info('Manual Reload Config.')
__config__.loadConfig(path=configfile)
Expand Down Expand Up @@ -1869,6 +1959,12 @@ def main_loop(ewd, op_is_valid, selection, np_is_valid, args):
menu_reload_config()
elif selection == 'p':
menu_print_config()
# PIXIV FANBOX
elif selection == 'f1':
menu_fanbox_download_supported_artist()
elif selection == 'f2':
menu_fanbox_download_by_artist_id()
# END PIXIV FANBOX
elif selection == '-all':
if not np_is_valid:
np_is_valid = True
Expand Down
77 changes: 77 additions & 0 deletions test.PixivModel_fanbox.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
#!/c/Python27/python.exe
# -*- coding: UTF-8 -*-
from __future__ import print_function

import sys
import os
import unittest

from PixivModelFanbox import Fanbox, FanboxArtist, FanboxPost
import PixivHelper


class TestPixivModel_Fanbox(unittest.TestCase):
currPath = unicode(os.path.abspath('.'))
PixivHelper.GetLogger()

def testFanboxSupportedArtist(self):
p = open('./test/Fanbox_supported_artist.json', 'r').read()
result = Fanbox(p)
self.assertIsNotNone(result)

self.assertEqual(len(result.supportedArtist), 3)
self.assertTrue(190026 in result.supportedArtist)
self.assertTrue(685000 in result.supportedArtist)
self.assertTrue(15521131 in result.supportedArtist)

def testFanboxArtistPosts(self):
p = open('./test/Fanbox_artist_posts.json', 'r').read()
result = FanboxArtist(15521131, p)
self.assertIsNotNone(result)

self.assertEqual(result.artist_id, 15521131)
self.assertTrue(result.hasNextPage)
self.assertTrue(len(result.nextUrl) > 0)
self.assertTrue(len(result.posts) > 0)

for post in result.posts:
self.assertFalse(post.is_restricted)

# post-136761
self.assertEqual(result.posts[0].post_id, 136761)
self.assertTrue(len(result.posts[0].title) > 0)
self.assertTrue(len(result.posts[0].coverImageUrl) > 0)
self.assertEqual(result.posts[0].type, "image")
self.assertEqual(len(result.posts[0].images), 5)

# post-132919
self.assertEqual(result.posts[2].post_id, 132919)
self.assertTrue(len(result.posts[2].title) > 0)
self.assertIsNone(result.posts[2].coverImageUrl)
self.assertEqual(result.posts[2].type, "text")
self.assertEqual(len(result.posts[2].images), 0)

# post-79695
self.assertEqual(result.posts[3].post_id, 79695)
self.assertTrue(len(result.posts[3].title) > 0)
self.assertIsNone(result.posts[3].coverImageUrl)
self.assertEqual(result.posts[3].type, "image")
self.assertEqual(len(result.posts[3].images), 4)

def testFanboxArtistPostsRestricted(self):
p = open('./test/Fanbox_artist_posts_restricted.json', 'r').read()
result = FanboxArtist(15521131, p)
self.assertIsNotNone(result)

self.assertEqual(result.artist_id, 15521131)
self.assertTrue(result.hasNextPage)
self.assertTrue(len(result.nextUrl) > 0)
self.assertTrue(len(result.posts) > 0)

for post in result.posts:
self.assertTrue(post.is_restricted)

if __name__ == '__main__':
# unittest.main()
suite = unittest.TestLoader().loadTestsFromTestCase(TestPixivModel_Fanbox)
unittest.TextTestRunner(verbosity=5).run(suite)
Loading

0 comments on commit d56bf81

Please sign in to comment.