From cac45722f4693293cc39e6e11fc2675cc4ae3264 Mon Sep 17 00:00:00 2001 From: Redcxx Date: Fri, 16 Aug 2019 16:46:08 +0800 Subject: [PATCH] making gui 10% --- gui/common.py | 24 ++ gui/factory.py | 48 +++ gui/lib/pikax/api/androidclient.py | 389 ++++++++++++++++++++ gui/lib/pikax/api/artwork.py | 147 ++++++++ gui/lib/pikax/api/defaultclient.py | 555 +++++++++++++++++++++++++++++ gui/lib/pikax/api/models.py | 128 +++++++ gui/lib/pikax/api/webclient.py | 363 +++++++++++++++++++ gui/lib/pikax/downloader.py | 76 ++++ gui/lib/pikax/exceptions.py | 98 +++++ gui/lib/pikax/items.py | 85 +++++ gui/lib/pikax/models.py | 407 +++++++++++++++++++++ gui/lib/pikax/params.py | 173 +++++++++ gui/lib/pikax/pikax.py | 237 ++++++++++++ gui/lib/pikax/processor.py | 44 +++ gui/lib/pikax/result.py | 34 ++ gui/lib/pikax/settings.py | 110 ++++++ gui/lib/pikax/user.py | 81 +++++ gui/lib/pikax/util.py | 339 ++++++++++++++++++ gui/login.py | 55 +++ gui/main.py | 22 ++ gui/menu.py | 56 +++ gui/models.py | 22 ++ gui/pikaxhandler.py | 26 ++ gui/rank.py | 13 + gui/search.py | 153 ++++++++ 25 files changed, 3685 insertions(+) create mode 100644 gui/common.py create mode 100644 gui/factory.py create mode 100644 gui/lib/pikax/api/androidclient.py create mode 100644 gui/lib/pikax/api/artwork.py create mode 100644 gui/lib/pikax/api/defaultclient.py create mode 100644 gui/lib/pikax/api/models.py create mode 100644 gui/lib/pikax/api/webclient.py create mode 100644 gui/lib/pikax/downloader.py create mode 100644 gui/lib/pikax/exceptions.py create mode 100644 gui/lib/pikax/items.py create mode 100644 gui/lib/pikax/models.py create mode 100644 gui/lib/pikax/params.py create mode 100644 gui/lib/pikax/pikax.py create mode 100644 gui/lib/pikax/processor.py create mode 100644 gui/lib/pikax/result.py create mode 100644 gui/lib/pikax/settings.py create mode 100644 gui/lib/pikax/user.py create mode 100644 gui/lib/pikax/util.py create mode 100644 gui/login.py create mode 100644 gui/main.py create mode 100644 gui/menu.py create mode 100644 gui/models.py create mode 100644 gui/pikaxhandler.py create mode 100644 gui/rank.py create mode 100644 gui/search.py diff --git a/gui/common.py b/gui/common.py new file mode 100644 index 0000000..60d4311 --- /dev/null +++ b/gui/common.py @@ -0,0 +1,24 @@ +import sys +from tkinter import END, NORMAL, DISABLED, Text, CENTER + + +def go_to_next_screen(src, dest): + pikax_handler = src.pikax_handler + master = src.frame.master + src.destroy() + dest(master, pikax_handler) + + +class StdoutRedirector: + def __init__(self, text_component): + self.text_component = text_component + + def write(self, string): + self.text_component.configure(state=NORMAL) + self.text_component.insert(END, string) + if isinstance(self.text_component, Text): + self.text_component.see(END) + self.text_component.configure(state=DISABLED) + + def flush(self): + pass diff --git a/gui/factory.py b/gui/factory.py new file mode 100644 index 0000000..99fa282 --- /dev/null +++ b/gui/factory.py @@ -0,0 +1,48 @@ +from tkinter import * + +# image = None +from tkinter import ttk + + +def make_button(master, text=''): + # global image + # image = PhotoImage(file='neurons.gif') + return Button(master=master, + text=text, + relief=RAISED, + state=DISABLED, + padx=10, + pady=2, + ) + + +def make_label(master, text=''): + return ttk.Label(master=master, + text=text) + + +def make_entry(master): + return ttk.Entry(master=master) + + +def make_frame(master): + return ttk.Frame(master=master) + + +def make_dropdown(master, default, choices): + dropdown = ttk.Combobox(master, values=choices, state='readonly') + dropdown.configure(width=17) + dropdown.set(default) + return dropdown + + +def make_text(master): + return Text(master, wrap=WORD, height=2, width=60, state=DISABLED) + + +def grid(component): + component.grid(padx=5, pady=5) + + +def pack(component): + component.pack(padx=5, pady=5) diff --git a/gui/lib/pikax/api/androidclient.py b/gui/lib/pikax/api/androidclient.py new file mode 100644 index 0000000..8c37281 --- /dev/null +++ b/gui/lib/pikax/api/androidclient.py @@ -0,0 +1,389 @@ +# +# special thanks @dazuling https://github.com/dazuling +# for explaining https://oauth.secure.pixiv.net/auth/token +# + + +import datetime +import time +import urllib.parse + +import requests + +from .defaultclient import DefaultAPIClient +from .models import APIUserInterface +from .. import params +from .. import util +from ..exceptions import ReqException, BaseClientException, ClientException, LoginError + +__all__ = ['AndroidAPIClient'] + + +class BaseClient: + # This class provide auto-refreshing headers to use for making requests + _auth_url = 'https://oauth.secure.pixiv.net/auth/token' + _headers = { + 'User-Agent': 'PixivAndroidApp/5.0.151 (Android 5.1.1; SM-N950N)', + 'App-OS': 'android', + 'App-OS-Version': '5.1.1', + 'App-Version': '5.0.151', + 'Host': 'app-api.pixiv.net', + } + + def __init__(self, username, password): + self._client_id = 'MOBrBDS8blbauoSck0ZfDbtuzpyT' + self._client_secret = 'lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj' + self._username = username + self._password = password + self._session = requests.Session() + self._headers = BaseClient._headers.copy() + + # set after login + self._access_token = None + self._access_token_start_time = None + self._refresh_token = None + self._token_type = None + + self.user_id = None + self._account = None + self._name = None + + # not used + self._device_token = None + self.mail = None + self.is_auth_mail = None + + try: + self._login() + except ReqException as e: + raise LoginError(str(e)) from e + + def _login(self): + + data = { + 'grant_type': 'password', + 'username': self._username, + 'password': self._password, + } + + res_data = self._auth_with_update(data) + + self._name = res_data['response']['user']['name'] + self._account = res_data['response']['user']['account'] + + self.user_id = res_data['response']['user']['id'] + self.mail = res_data['response']['user']['mail_address'] + self.is_auth_mail = res_data['response']['user']['is_mail_authorized'] + + def _auth_with_update(self, extra_data): + data = { + 'client_id': self._client_id, + 'client_secret': self._client_secret, + 'get_secure_url': 1, + # https://github.com/dazuling/pixiv-api/blob/master/pixivapi/client.py#L154 + **extra_data + } + + res = util.req( + url=BaseClient._auth_url, + req_type='post', + session=self._session, + data=data, + ).json() + + self._access_token_start_time = time.time() + self._access_token = res['response']['access_token'] + self._refresh_token = res['response']['refresh_token'] + self._token_type = res['response']['token_type'] + self._access_token_time_out = int(res['response']['expires_in']) # time out for refresh token in seconds + self._device_token = res['response']['device_token'] + self._headers.update({'Authorization': f'{self._token_type.title()} {self._access_token}'}) + + return res + + def _update_token_if_outdated(self): + if self._is_access_token_outdated(): + self._update_access_token() + + def _is_access_token_outdated(self): + time_ahead = 30 # return True if need refresh within 30s + return (time.time() - self._access_token_start_time + time_ahead) > self._access_token_time_out + + def _update_access_token(self): + data = { + 'grant_type': 'refresh_token', + 'refresh_token': self._refresh_token + } + try: + self._auth_with_update(data) + except ReqException as e: + raise BaseClientException('Failed update access token') from e + + @property + def headers(self): + self._update_token_if_outdated() + return self._headers + + +class FunctionalBaseClient(BaseClient): + # This class provide utilities for the real job, e.g. search/accessing user ... + _host = 'https://app-api.pixiv.net' + _search_url = _host + '/v1/search/{type}?' + _following_url = _host + '/v1/user/following?' + _illust_creation_url = _host + '/v1/user/illusts?' + _collection_url = _host + '/v1/user/bookmarks/{collection_type}?' + _tagged_collection_url = _host + '/v1/user/bookmark-tags/{collection_type}?' + + def __init__(self, username, password): + super().__init__(username, password) + self._name = None + self._account = None + + @classmethod + def _get_search_start_url(cls, keyword, search_type, match, sort, search_range): + cls._check_params(match=match, sort=sort, search_range=search_range) + if search_type and not params.SearchType.is_valid(search_type): + raise BaseClientException(f'search type must be type of {params.SearchType}') + param = {'word': str(keyword), 'search_target': match.value, 'sort': sort.value} + + if search_range: + if params.Range.is_valid(search_range): + search_range = search_range.value + today = datetime.date.today() + param['start_date'] = str(today) + param['end_date'] = str(today - search_range) + + encoded_params = urllib.parse.urlencode(param) + return cls._search_url.format(type=search_type.value) + encoded_params + + @classmethod + def _get_bookmarks_start_url(cls, bookmark_type, req_params, tagged): + if bookmark_type and not params.BookmarkType.is_valid(bookmark_type): + raise BaseClientException(f'bookmark type: {bookmark_type} is not type of {params.BookmarkType}') + + if tagged: + collection_url = cls._tagged_collection_url.format(collection_type=bookmark_type.value) + else: + collection_url = cls._collection_url.format(collection_type=bookmark_type.value) + + encoded_params = urllib.parse.urlencode(req_params) + return collection_url + encoded_params + + @classmethod + def _get_creations_start_url(cls, req_params): + encoded_params = urllib.parse.urlencode(req_params) + return cls._illust_creation_url + encoded_params + + @staticmethod + def _check_params(match=None, sort=None, search_range=None, restrict=None): + if match and not params.Match.is_valid(match): + raise BaseClientException(f'match: {match} is not type of {params.Match}') + if sort and not params.Sort.is_valid(sort): + raise BaseClientException(f'sort: {sort} is not type of {params.Sort}') + if search_range and not params.Range.is_valid(search_range): + raise BaseClientException(f'search_range: {search_range} is not type of {params.Range}') + if restrict and not params.Restrict.is_valid(restrict): + raise BaseClientException(f'restrict: {restrict} is not type of {params.Restrict}') + + def req(self, url, req_params=None): + return util.req(url=url, headers=self.headers, params=req_params) + + def _get_ids(self, next_url, limit, id_type): + if limit: + limit = int(limit) + data_container_name = params.Type.get_response_container_name(id_type.value) + ids_collected = [] + while next_url is not None and (not limit or len(ids_collected) < limit): + res_data = self.req(next_url).json() + if id_type is params.Type.USER: + ids_collected += [item['user']['id'] for item in res_data[data_container_name]] + else: + ids_collected += [item['id'] for item in res_data[data_container_name]] + next_url = res_data['next_url'] + ids_collected = list(set(ids_collected)) + if limit: + ids_collected = util.trim_to_limit(ids_collected, limit) + return ids_collected + + def get_bookmarks(self, bookmark_type, limit, restrict, tagged, user_id): + self._check_params(restrict=restrict) + + req_params = { + 'user_id': int(user_id), + 'restrict': restrict.value + } + start_url = self._get_bookmarks_start_url(bookmark_type, req_params, tagged=tagged) + if bookmark_type is params.BookmarkType.ILLUST_OR_MANGA: + bookmark_type = params.Type.ILLUST + return self._get_ids(start_url, limit=limit, id_type=bookmark_type) + + def get_creations(self, creation_type, limit, user_id): + if not params.CreationType.is_valid(creation_type): + raise ClientException( + f'creation type must be type of {params.CreationType}') + + req_params = { + 'user_id': int(user_id), + 'type': creation_type.value + } + + start_url = self._get_creations_start_url(req_params=req_params) + return self._get_ids(start_url, limit=limit, id_type=creation_type) + + +class AndroidAPIClient(FunctionalBaseClient, DefaultAPIClient): + # This class will be used by Pikax as api + + class User(APIUserInterface): + _details_url = 'https://app-api.pixiv.net/v1/user/detail?' + + # This class represent other user + def __init__(self, client, user_id): + self.client = client + self.user_id = user_id + self._config() + + def _config(self): + req_params = { + 'user_id': self.id + } + try: + data = self.client.req(url=self._details_url + urllib.parse.urlencode(req_params)).json() + self._account = data['user']['account'] + self._name = data['user']['name'] + except (ReqException, KeyError) as e: + from ..exceptions import APIUserError + raise APIUserError(f'Failed to config user details: {e}') + + def bookmarks(self, limit=None, bookmark_type=params.BookmarkType.ILLUST_OR_MANGA, tagged=None): + return self.client.get_bookmarks(bookmark_type=bookmark_type, limit=limit, restrict=params.Restrict.PUBLIC, + tagged=tagged, user_id=self.user_id) + + def illusts(self, limit=None): + return self.client.get_creations(creation_type=params.CreationType.ILLUST, limit=limit, + user_id=self.user_id) + + def mangas(self, limit=None): + return self.client.get_creations(creation_type=params.CreationType.MANGA, limit=limit, user_id=self.user_id) + + @property + def id(self): + return self.user_id + + @property + def account(self): + return self._account + + @property + def name(self): + return self._name + + def __init__(self, username, password): + FunctionalBaseClient.__init__(self, username, password) + + def search(self, keyword='', search_type=params.SearchType.ILLUST_OR_MANGA, match=params.Match.PARTIAL, + sort=params.Sort.DATE_DESC, + search_range=None, limit=None): + # if params.user is passed in as type, + # only keyword is considered + + start_url = self._get_search_start_url(keyword=keyword, search_type=search_type, match=match, sort=sort, + search_range=search_range) + ids = self._get_ids(start_url, limit=limit, id_type=search_type) + + # XXX attempt to fix user input keyword if no search result returned? + # if not ids: + # auto_complete_keyword = self._get_keyword_match(word=keyword) + # if auto_complete_keyword: + # return self.search(keyword=auto_complete_keyword, type=type, match=match, sort=sort, + # range=range, limit=limit, r18=r18, r18g=r18g) + + return ids + + def rank(self, limit=None, date=format(datetime.date.today(), '%Y%m%d'), content=params.Content.ILLUST, + rank_type=params.RankType.DAILY): + return super().rank(limit=limit, date=date, content=content, rank_type=rank_type) + + def bookmarks(self, limit=None, bookmark_type: params.BookmarkType = params.BookmarkType.ILLUST_OR_MANGA, + restrict: params.Restrict = params.Restrict.PUBLIC): + return self.get_bookmarks(bookmark_type=bookmark_type, limit=limit, restrict=restrict, tagged=False, # XXX + user_id=self.user_id) + + def illusts(self, limit=None): + return self.get_creations(creation_type=params.CreationType.ILLUST, limit=limit, user_id=self.user_id) + + def mangas(self, limit=None): + return self.get_creations(creation_type=params.CreationType.MANGA, limit=limit, user_id=self.user_id) + + def visits(self, user_id): + return AndroidAPIClient.User(self, user_id) + + @property + def account(self): + return self._account + + @property + def name(self): + return self._name + + @property + def id(self): + return self.user_id + + +def test(): + from .. import settings + + print('Testing AndroidClient') + + client = AndroidAPIClient(settings.username, settings.password) + + ids = client.search(keyword='arknights', limit=242, sort=params.Sort.DATE_DESC, match=params.Match.ANY, + search_range=params.Range.A_YEAR) + assert len(ids) == 242, len(ids) + + ids = client.bookmarks(limit=30) + assert len(ids) == 30, len(ids) + + ids = client.mangas(limit=0) + assert len(ids) == 0, len(ids) + + ids = client.search(keyword='arknights', limit=234, sort=params.Sort.DATE_DESC, + search_type=params.SearchType.ILLUST_OR_MANGA, + match=params.Match.EXACT, + search_range=params.Range.A_MONTH) + assert len(ids) == 234 + + ids = client.rank(rank_type=params.RankType.ROOKIE, date=datetime.date.today(), content=params.Content.MANGA) + assert len(ids) == 100, len(ids) + + user_id = 38088 + user = client.visits(user_id=user_id) + user_illust_ids = user.illusts() + assert len(user_illust_ids) == 108, len(user_illust_ids) + + user_manga_ids = user.mangas() + assert len(user_manga_ids) == 2, len(user_manga_ids) + + print('Successfully tested Android Client') + + +def main(): + test() + # while True: + # try: + # res = util.req(url=url, session=client._session, headers=client._headers, params=params) + # except ReqException as e: + # print(e) + # print('Failed request, trying to refresh token ...') + # client._update_access_token() + # print('+' * 50) + # continue + # print(res) + # print(time.time() - client._access_token_start_time) + # print('=' * 10) + # time.sleep(10) + + +if __name__ == '__main__': + main() diff --git a/gui/lib/pikax/api/artwork.py b/gui/lib/pikax/api/artwork.py new file mode 100644 index 0000000..3ab1cc6 --- /dev/null +++ b/gui/lib/pikax/api/artwork.py @@ -0,0 +1,147 @@ +import os +import re + +from .models import Artwork +from .. import util, settings +from ..exceptions import ReqException, ArtworkError + +__all__ = ['Illust'] + + +class Illust(Artwork): + """ + + extra properties + + """ + _referer_url = 'https://www.pixiv.net/member_illust.php?mode=medium&illust_id=' + _details_url = 'https://www.pixiv.net/ajax/illust/' + _headers = { + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/75.0.3770.100 Safari/537.36' + } + + def __init__(self, illust_id): + # properties, set after generate details is called + self._views = None + self._bookmarks = None + self._title = None + self._author = None + self._likes = None + + # iterator use, set after generate download data is called + self.__download_urls = None + + # not used, set after generate details is called + self.__original_url_template = None + self.__comments = None + + # internal uses + self.__page_count = None + self._details_url = Illust._details_url + str(illust_id) + self._headers = Illust._headers.copy() + self._headers['referer'] = Illust._referer_url + str(illust_id) + super().__init__(illust_id) + + def config(self): + try: + illust_data = util.req(req_type='get', url=self._details_url, log_req=False).json() + illust_data = illust_data['body'] + + # properties + self._views = illust_data['viewCount'] + self._bookmarks = illust_data['bookmarkCount'] + self._likes = illust_data['likeCount'] + self._title = illust_data['illustTitle'] + self._author = illust_data['userName'] + + self.__original_url_template = illust_data['urls']['original'] + self.__original_url_template = re.sub(r'(?<=_p)\d', '{page_num}', self.__original_url_template) + self.__comments = illust_data['commentCount'] + self.__page_count = illust_data['pageCount'] + + self.__generate_download_data() + except (ReqException, KeyError) as e: + print(e) + raise ArtworkError(f'Failed to configure artwork of id: {self.id}') from e + + def _get_download_url(self, page_num): + return self.__original_url_template.format(page_num=page_num) + + def _get_download_filename(self, download_url, folder=None): + id_search = re.search(r'(\d{8}_p\d.*)', download_url) + illust_signature = id_search.group(1) if id_search else download_url + filename = str(self.author) + '_' + str(illust_signature) + if folder is not None: + filename = os.path.join(util.clean_filename(folder), filename) + return util.clean_filename(filename) + + def __generate_download_data(self): + self.__download_urls = [] + curr_page = 0 + + while curr_page < self.__page_count: + if self._reached_limit_in_settings(curr_page): + break + self.__download_urls.append(self._get_download_url(curr_page)) + curr_page += 1 + + def __getitem__(self, index): + download_url = self.__download_urls[index] + filename = self._get_download_filename(download_url) + + return Artwork.DownloadStatus.OK, (download_url, self._headers), filename + + def __len__(self): + return len(self.__download_urls) + + @property + def bookmarks(self): + return self._bookmarks + + @property + def views(self): + return self._views + + @property + def author(self): + return self._author + + @property + def title(self): + return self._title + + @property + def likes(self): + return self._likes + + @staticmethod + def _reached_limit_in_settings(current): + if settings.MAX_PAGES_PER_ARTWORK: + if current >= settings.MAX_PAGES_PER_ARTWORK: + return True + return False + + +def test(): + print('Testing Illust Artwork') + from .androidclient import AndroidAPIClient + from .. import settings + client = AndroidAPIClient(settings.username, settings.password) + user = client.visits(user_id=2957827) + illust_ids = user.illusts(limit=15) + artworks = [Illust(illust_id) for illust_id in illust_ids] + for artwork in artworks: + artwork.config() + for status, content, filename in artwork: + print(status, filename) + + print('Successfully tested Illust Artwork') + + +def main(): + test() + + +if __name__ == '__main__': + main() diff --git a/gui/lib/pikax/api/defaultclient.py b/gui/lib/pikax/api/defaultclient.py new file mode 100644 index 0000000..e3addf1 --- /dev/null +++ b/gui/lib/pikax/api/defaultclient.py @@ -0,0 +1,555 @@ +import datetime +import re +import time +from typing import List, Union + +from .models import APIPagesInterface, APIUserInterface, APIAccessInterface +from .. import params, settings +from .. import util +from ..exceptions import ReqException, SearchError, RankError, UserError + + +__all__ = ['DefaultAPIClient', 'DefaultAPIUser'] + + +class DefaultIllustSearch: + """Representing the search page in pixiv.net + + **Functions** + :func search: Used to search in pixiv.net + + """ + _search_popularity_postfix = u'users入り' + + def __init__(self): + pass + + @classmethod + def _set_params(cls, search_type, dimension, match, sort, search_range): + + search_params = dict() + if search_type: # default match all type + if not params.SearchType.is_valid(search_type): + search_params['type'] = search_type.value + + if dimension: # default match all ratios + if params.Dimension.is_valid(dimension): + search_params['ratio'] = dimension.value + else: + raise SearchError(f'dimension type: {dimension} is not type of {params.Dimension}') + + if match: # default match if contain tags + if not params.Match.is_valid(match): + raise SearchError(f'match: {match} is not type of {params.Match}') + + if match is params.Match.PARTIAL: # this is default + pass + elif match is params.Match.EXACT: # specified tags only + search_params['s_mode'] = 's_tag_full' + elif match == params.Match.ANY: + search_params['s_mode'] = 's_tc' + + if sort: + if not params.Sort.is_valid(sort): + raise SearchError(f'sort: {sort} is not type of {params.Sort}') + + if sort is params.Sort.DATE_DESC: + search_params['order'] = 'date_d' + elif sort is params.Sort.DATE_ASC: + search_params['order'] = 'date' + + if search_range: + if params.Range.is_valid(search_range): + search_range = search_range.value + if isinstance(search_range, datetime.timedelta): + today = datetime.date.today() + search_params['ecd'] = str(today) + search_params['scd'] = str(today - search_range) + else: + raise SearchError(f'Invalid range type: {search_range}') + + return search_params + + @classmethod + def _search_all_popularities_in_list(cls, search_params, keyword, limit, session): + ids = [] + total_limit = limit + for popularity in settings.SEARCH_POPULARITY_LIST: + ids += cls._search(search_params=search_params, keyword=keyword, limit=limit, popularity=popularity, + session=session) + if total_limit: + num_of_ids_sofar = len(ids) + if num_of_ids_sofar > total_limit: + ids = util.trim_to_limit(ids, total_limit) + break + else: + limit = total_limit - num_of_ids_sofar + return ids + + @classmethod + def search(cls, keyword, limit=None, search_type=None, dimension=None, match=None, popularity=None, sort=None, + search_range=None, session=None): + """Used to search in pixiv.net + + **Parameters** + :param session: + :param search_range: + :param sort: + :param keyword: + a space separated of tags, used for search + :type keyword: + str + + :param limit: + number of artworks is trimmed to this number if too many, may not be enough + :type limit: + int or None(default) + + :param search_type: + type of artworks, + 'illust' | 'manga', default any + :type search_type: + str or None(default) + + :param dimension: + dimension of the artworks, 'vertical' | 'horizontal' | 'square', default any + :type dimension: + str or None(default) + + :param match: + define the way of matching artworks with a artwork, + 'strict_tag' matches when any keyword is same as a tag in the artwork + 'loose' matches when any keyword appears in title, description or tags of the artwork + default matches when any keyword is part of a tag of the artwork + :type match: + str or None(default) + + :param popularity: + this is based on a pixiv search trick to return popular results for non-premium users, + eg, pixiv automatically adds a 1000users入り tag when a artwork has 1000 likes + when popularity is given, the str ' ' + popularity + 'users入り' is added to keyword string, + common popularity of 100, 500, 1000, 5000, 10000, 20000 is strongly suggested, since pixiv does + not add tag for random likes such as 342users入り + when str 'popular' is given, it will search for all results with users入り tag in + [20000, 10000, 5000, 1000, 500] + note that 'popular' is the only string accepted + :type popularity: + int or str or None(default) + + **Returns** + :return: a list of Artwork Object + :rtype: python list + + + **Raises** + :raises SearchError: if invalid order, mode or dimension is given + + + **See Also** + :class: items.Artwork + + """ + + # for recording + start = time.time() + + # setting parameters + search_params = cls._set_params(search_type=search_type, dimension=dimension, match=match, sort=sort, + search_range=search_range) + + if not keyword: + keyword = '' + + # search starts + if popularity == 'popular': + ids = cls._search_all_popularities_in_list(search_params=search_params, keyword=keyword, limit=limit, + session=session) + else: + ids = cls._search(search_params=search_params, keyword=keyword, limit=limit, popularity=popularity, + session=session) + + # log ids found + util.log('Found', str(len(ids)), 'ids for', keyword, 'in', str(time.time() - start) + 's') + + return ids + # # build artworks from ids + # artworks = util.generate_artworks_from_ids(ids) + # + # return artworks + + @classmethod + def _search(cls, search_params, keyword, popularity, limit, session): + curr_page = 1 + ids_so_far = [] + url = 'https://www.pixiv.net/search.php?' + search_regex = r'(\d{8})_p\d' + while True: + # get a page's ids + search_params['p'] = curr_page + search_params['word'] = str(keyword) + if popularity is not None: + search_params['word'] += ' ' + str(popularity) + cls._search_popularity_postfix + util.log('Searching illust id for params:', search_params, 'at page:', curr_page) + try: + err_msg = 'Failed getting ids from params ' + str(search_params) + ' page: ' + str(curr_page) + results = util.req(url=url, params=search_params, session=session, + err_msg=err_msg, log_req=False) + except ReqException as e: + util.log(str(e), error=True, save=True) + if curr_page == 1: + util.log('Theres no result found for input', inform=True, save=True) + else: + util.log('End of search at page: ' + str(curr_page), inform=True, save=True) + return ids_so_far + + ids = re.findall(search_regex, results.text) + + # set length of old ids and new ids, + # use later to check if reached end of all pages + old_len = len(ids_so_far) + ids_so_far += ids + ids_so_far = list(set(ids_so_far)) + new_len = len(ids_so_far) + + # if limit is specified, check if reached limited number of items + if limit is not None: + if limit == new_len: + return ids_so_far + elif limit < new_len: + return util.trim_to_limit(ids_so_far, limit=limit) + # limit has not reached + + # now check if any new items is added + if old_len == new_len: # if no new item added, end of search pages + if limit is not None: # if limit is specified, it means search ended without meeting user's limit + util.log('Search did not return enough items for limit:', new_len, '<', limit, inform=True, + save=True) + return ids_so_far + + # search next page + curr_page += 1 + + +class DefaultRank: + """Representing ranking page in pixiv.net + + **Functions** + :func rank: used to get artworks in rank page in pixiv.net + + """ + + url = 'https://www.pixiv.net/ranking.php?' + + def __init__(self): + pass + + @classmethod + def _check_inputs(cls, content, rank_type): + if content is params.Content.ILLUST: + allowed = [params.RankType.DAILY, params.RankType.MONTHLY, params.RankType.WEEKLY, params.RankType.ROOKIE] + if rank_type not in allowed: + raise RankError('Illust content is only available for type in', allowed) + + @classmethod + def _set_params(cls, content, date, rank_type): + rank_params = dict() + + rank_params['format'] = 'json' + + if rank_type: + if params.RankType.is_valid(rank_type): + rank_params['mode'] = rank_type.value + else: + raise RankError(f'rank type: {rank_type} is not type of {params.RankType}') + + if content: + if params.Content.is_valid(content): + rank_params['content'] = content.value + else: + raise RankError(f'content: {content} is not type of {params.Content}') + + if date: + if isinstance(date, str): + rank_params['date'] = date + elif isinstance(date, datetime.date): + rank_params['date'] = format(date, '%Y%m%d') + elif isinstance(date, params.Date): + rank_params['date'] = format(date.value, '%Y%m%d') + else: + raise RankError(f'Invalid date: {date}') + + if rank_params['date'] == format(datetime.date.today(), '%Y%m%d'): + del rank_params['date'] # pixiv always shows previous day rank for today + + return rank_params + + @classmethod + def _rank(cls, rank_params, limit): + ids = [] + page_num = 0 + while True: + page_num += 1 + rank_params['p'] = page_num + try: + res = util.req(url=cls.url, params=rank_params).json() + except ReqException as e: + util.log(str(e), error=True, save=True) + util.log('End of rank at page:', page_num, inform=True, save=True) + break + if 'error' in res: + util.log('End of page while searching', str(rank_params) + '. Finished') + break + else: + ids += [content['illust_id'] for content in res['contents']] + + # check if number of ids reached requirement + if limit: + num_of_ids_found = len(ids) + if limit == num_of_ids_found: + break + elif limit < num_of_ids_found: + ids = util.trim_to_limit(ids, limit) + break + + return ids + + @classmethod + def rank(cls, rank_type, content, limit=None, date=None): + """Used to get artworks from pixiv ranking page + + **Parameters** + :param rank_type: + type of ranking as in pixiv.net, + 'daily' | 'weekly' | 'monthly' | 'rookie' | 'original' | 'male' | 'female', default daily + :type rank_type: + params.Rank + + :param limit: + number of artworks to return, may not be enough, default all + :type limit: + int or None + + :param date: + the date when ranking occur, + if string given it must be in 'yyyymmdd' format + eg. given '20190423' and mode daily will return the daily ranking of pixiv on 2019 April 23 + eg. given '20190312' and mode monthly will return the monthly ranking from 2019 Feb 12 to 2019 March 12 + default today + :type date: + Datetime or str or None + + :param content: + type of artwork to return, + 'illust' | 'manga', default 'illust' + :type content: + params.Content + + **Returns** + :return: a list of artworks + :rtype: list + + """ + + # some combinations are not allowed + cls._check_inputs(content=content, rank_type=rank_type) + + # set paramters + rank_params = cls._set_params(content=content, date=date, rank_type=rank_type) + + # rank starts + ids = cls._rank(rank_params=rank_params, limit=limit) + + # if limit is specified, check if met + if limit: + num_of_ids_found = len(ids) + if num_of_ids_found < limit: + util.log('Items found in ranking is less than requirement:', num_of_ids_found, '<', limit, inform=True) + + return ids + + +class DefaultAPIUser(APIUserInterface): + # for retrieving details + _user_details_url = 'https://www.pixiv.net/touch/ajax/user/details?' # param id + _self_details_url = 'https://www.pixiv.net/touch/ajax/user/self/status' # need login session + + # for retrieving contents + _content_url = 'https://www.pixiv.net/touch/ajax/user/illusts?' + _bookmarks_url = 'https://www.pixiv.net/touch/ajax/user/bookmarks?' + _illusts_url = 'https://www.pixiv.net/touch/ajax/illust/user_illusts?user_id={user_id}' + + _profile_url = 'https://www.pixiv.net/ajax/user/{user_id}/profile/all' + + # for settings + _settings_url = 'https://www.pixiv.net/setting_user.php' + # user language for settings + _user_lang_dict = { + 'zh': u'保存', + 'zh_tw': u'保存', + 'ja': u'変更', + 'en': u'Update', + 'ko': u'변경' + } + + def __init__(self, user_id, session=None): + self._id = user_id + self._session = session + self._config() + + def _config(self): + # get information from user id + details_params = dict({'id': self.id}) + try: + data = util.req(url=self._user_details_url, params=details_params, session=self._session).json() + except ReqException as e: + util.log(str(e), error=True, save=True) + raise UserError('Failed to load user information') + + # save user information, not used yet, for filter in the future + data = data['user_details'] + self._id = data['user_id'] + self._account = data['user_account'] + self._name = data['user_name'] + self.title = data['meta']['title'] + self.description = data['meta']['description'] + self.follows = data['follows'] + + # init user's contents + try: + data = util.req(url=self._profile_url.format(user_id=self.id), session=self._session).json() + self._illust_ids = list(data['body']['illusts'].keys()) if data['body']['illusts'] else [] + self._manga_ids = list(data['body']['manga'].keys()) if data['body']['manga'] else [] + except (ReqException, KeyError) as e: + util.log(str(e), error=True, save=True) + raise UserError(f'Failed to load user creations: {e}') + + def illusts(self, limit=None): + """Returns illustrations uploaded by this user + + **Parameters** + :param limit: + limit the amount of illustrations found, if exceed + :type limit: + int or None + + :return: the results of attempting to retrieve this user's uploaded illustrations + :rtype: PixivResult Object + + """ + return util.trim_to_limit(self._illust_ids, limit=limit) + + def mangas(self, limit=None): + """Returns mangas uploaded by this user + + **Parameters** + :param limit: + limit the amount of mangas found, if exceed + :type limit: + int or None + + :return: the results of attempting to retrieve this user's uploaded mangas + :rtype: PixivResult Object + + """ + return util.trim_to_limit(self._manga_ids, limit=limit) + + def bookmarks(self, limit: int = None, bookmark_type: params.Type = params.Type.ILLUST, + restrict: params.Restrict = params.Restrict.PUBLIC) -> List[int]: + raise NotImplementedError('Bookmark is inaccessible without logging into Pixiv') + + @property + def account(self): + return self._account + + @property + def name(self): + return self._name + + @property + def id(self): + return self._id + + +class DefaultAPIClient(APIPagesInterface, APIUserInterface, APIAccessInterface): + + def __init__(self, session=None): + self._session = session + + def search(self, keyword: str = '', search_type: params.SearchType = params.SearchType.ILLUST_OR_MANGA, + match: params.Match = params.Match.PARTIAL, sort: params.Sort = params.Sort.DATE_DESC, + search_range: Union[datetime.timedelta, params.Range] = None, limit: int = None) -> List[int]: + if search_type is params.SearchType.ILLUST_OR_MANGA: + return DefaultIllustSearch.search(keyword=keyword, search_type=search_type, match=match, sort=sort, + search_range=search_range, limit=limit, session=self._session) + + def rank(self, limit: int = None, date: Union[str, datetime.date] = format(datetime.date.today(), '%Y%m%d'), + content: params.Content = params.Content.ILLUST, rank_type: params.RankType = params.RankType.DAILY) -> \ + List[int]: + return DefaultRank.rank(limit=limit, date=date, content=content, rank_type=rank_type) + + def visits(self, user_id: int) -> APIUserInterface: + return DefaultAPIUser(user_id=user_id, session=self._session) + + def bookmarks(self, limit: int = None, bookmark_type: params.Type = params.Type.ILLUST, + restrict: params.Restrict = params.Restrict.PUBLIC) -> List[int]: + raise NotImplementedError('Bookmark is inaccessible in Pixiv without login') + + def illusts(self, limit: int = None) -> List[int]: + raise NotImplementedError('Illust is inaccessible in Pixiv without login') + + def mangas(self, limit: int = None) -> List[int]: + raise NotImplementedError('Manga is inaccessible in Pixiv without login') + + @property + def account(self): + raise NotImplementedError('Account is inaccessible in Pixiv without login') + + @property + def id(self): + raise NotImplementedError('Id is inaccessible in Pixiv without login') + + @property + def name(self): + raise NotImplementedError('Name is inaccessible in Pixiv without login') + + +def test(): + user = DefaultAPIUser(user_id=9665193) + client = DefaultAPIClient() + ids = user.illusts(limit=15) + assert len(ids) == 15, len(ids) + + ids = user.mangas(limit=1) + assert len(ids) == 0 or len(ids) == 1, len(ids) + ids = client.search(keyword='arknights', limit=234, sort=params.Sort.DATE_DESC, + search_type=params.SearchType.ILLUST_OR_MANGA, + match=params.Match.EXACT, + search_range=params.Range.A_MONTH) + assert len(ids) == 234, len(ids) + + ids = client.rank(rank_type=params.RankType.DAILY, limit=400, date=datetime.date.today(), + content=params.Content.ILLUST) + assert len(ids) == 400, len(ids) + + user_id = 38088 + user = client.visits(user_id=user_id) + print('id', user.id) + print('account', user.account) + print('name', user.name) + user_illust_ids = user.illusts(limit=65) + assert len(user_illust_ids) == 65, len(user_illust_ids) + + user_manga_ids = user.mangas(limit=2) + assert len(user_manga_ids) == 2 + + print('Successfully tested default client') + + +def main(): + import sys + sys.stdout.reconfigure(encoding='utf-8') + test() + + +if __name__ == '__main__': + main() diff --git a/gui/lib/pikax/api/models.py b/gui/lib/pikax/api/models.py new file mode 100644 index 0000000..a31e7bd --- /dev/null +++ b/gui/lib/pikax/api/models.py @@ -0,0 +1,128 @@ +import datetime +import enum +import os +from multiprocessing.dummy import Pool +from typing import List, Tuple, Union, Type, Any + +from .. import params, util +from ..exceptions import ArtworkError + + +class APIUserInterface: + + def bookmarks(self, limit: int = None, bookmark_type: params.BookmarkType = params.BookmarkType.ILLUST_OR_MANGA, + restrict: params.Restrict = params.Restrict.PUBLIC) -> List[int]: + raise NotImplementedError + + def illusts(self, limit: int = None) -> List[int]: raise NotImplementedError + + def mangas(self, limit: int = None) -> List[int]: raise NotImplementedError + + @property + def id(self): raise NotImplementedError + + @property + def name(self): raise NotImplementedError + + @property + def account(self): raise NotImplementedError + + +class APIAccessInterface: + def visits(self, user_id: int) -> APIUserInterface: + raise NotImplementedError + + +class APIPagesInterface: + + def search(self, keyword: str = '', + search_type: params.Type = params.Type.ILLUST, + match: params.Match = params.Match.PARTIAL, + sort: params.Sort = params.Sort.DATE_DESC, + search_range: Union[datetime.timedelta, params.Range] = None, + limit: int = None) -> List[int]: raise NotImplementedError + + def rank(self, + rank_type: params.RankType = params.RankType.DAILY, + content: params.Content = params.Content.ILLUST, + date: Union[str, datetime.date] = format(datetime.date.today(), '%Y%m%d'), + limit: int = None) -> List[int]: raise NotImplementedError + + +class Artwork: + + def __init__(self, artwork_id): + self.id = artwork_id + self.config() + + @property + def bookmarks(self): raise NotImplementedError + + @property + def views(self): raise NotImplementedError + + @property + def author(self): raise NotImplementedError + + @property + def title(self): raise NotImplementedError + + @property + def likes(self): raise NotImplementedError + + class DownloadStatus(enum.Enum): + OK = '[OK]' + SKIPPED = '[skipped]' + FAILED = '' + + # return download status, content, filename + def __getitem__(self, index) -> Tuple[DownloadStatus, Any, str]: raise NotImplementedError + + # return num of pages + def __len__(self): raise NotImplementedError + + # set variables, raises ReqException if fails + def config(self): + raise NotImplementedError + + +class BaseIDProcessor: + + def __init__(self): + self.type_to_function = { + params.ProcessType.MANGA: self.process_mangas, + params.ProcessType.ILLUST: self.process_illusts, + } + + def process_illusts(self, ids: List[int]) -> Tuple[List[Artwork], List[int]]: + raise NotImplementedError + + def process_mangas(self, ids: List[int]) -> Tuple[List[Artwork], List[int]]: + raise NotImplementedError + + def process(self, ids: List[int], process_type: params.ProcessType) -> Tuple[List[Artwork], List[int]]: + if not params.ProcessType.is_valid(process_type): + from ..exceptions import ProcessError + raise ProcessError(f'process type: {process_type} is not type of {params.ProcessType}') + + return self.type_to_function[process_type](ids) + + @staticmethod # param cls is pass in as argument + def _general_processor(cls: Type[Artwork], item_ids: List[int]) -> Tuple[List[Artwork], List[int]]: + util.log('Processing artwork ids', start=os.linesep, inform=True) + total = len(item_ids) + successes = [] + fails = [] + pool = Pool() + + def process_item(itemid): + try: + successes.append(cls(itemid)) + except ArtworkError: + fails.append(itemid) + + for index, item_id in enumerate(pool.imap_unordered(process_item, item_ids)): + util.print_progress(index + 1, total) + msg = f'expected: {total} | success: {len(successes)} | failed: {len(fails)}' + util.print_done(msg) + return successes, fails diff --git a/gui/lib/pikax/api/webclient.py b/gui/lib/pikax/api/webclient.py new file mode 100644 index 0000000..d60cfa1 --- /dev/null +++ b/gui/lib/pikax/api/webclient.py @@ -0,0 +1,363 @@ +import datetime +import os +import pickle +import re +from typing import Union, List + +from .. import params +from .. import util, settings +from ..api.defaultclient import DefaultAPIClient, DefaultAPIUser +from ..exceptions import ReqException, LoginError, APIUserError + +__all__ = ['WebAPIClient'] + + +class BaseClient: + _login_check_url = 'https://www.pixiv.net/touch/ajax/user/self/status' + + def __init__(self): + self._session = util.new_session() + self.cookies_file = settings.COOKIES_FILE + + def _check_is_logged(self): + status_json = util.req(url=self._login_check_url, session=self._session).json() + return status_json['body']['user_status']['is_logged_in'] + + def _save_cookies(self): + + if os.path.isfile(self.cookies_file): + util.log(f'Rewriting local cookie file: {self.cookies_file}') + else: + util.log(f'Saving cookies to local file: {self.cookies_file}') + + with open(self.cookies_file, 'wb') as file: + pickle.dump(self._session.cookies, file) + + def _login(self, *args): + raise NotImplementedError + + +class AccountClient(BaseClient): + _login_url = 'https://accounts.pixiv.net/api/login?' + _post_key_url = 'https://accounts.pixiv.net/login?' + + def __init__(self): + super().__init__() + + def _login(self, username, password): + postkey = self._get_postkey() + + data = { + 'password': password, + 'pixiv_id': username, + 'post_key': postkey, + } + login_params = { + 'lang': 'en' + } + + util.log('Sending requests to attempt login ...') + + try: + util.req(req_type='post', session=self._session, url=self._login_url, data=data, params=login_params) + except ReqException as e: + raise LoginError(f'Failed to send login request: {e}') + + util.log('Login request sent to Pixiv') + if self._check_is_logged(): + self._save_cookies() + else: + raise LoginError('Login Request is not accepted') + + def _get_postkey(self): + try: + pixiv_login_page = util.req(session=self._session, url=self._post_key_url) + post_key = re.search(r'post_key" value="(.*?)"', pixiv_login_page.text).group(1) + util.log(f'Post key successfully retrieved: {post_key}') + return post_key + except (ReqException, AttributeError) as e: + raise LoginError(f'Failed to find post key: {e}') + + +class CookiesClient(BaseClient): + + def __init__(self): + super().__init__() + + def _login(self): + try: + self._local_cookies_login() + except LoginError: + try: + self._user_cookies_login() + except LoginError: + raise LoginError('Cookies Login failed') + + def _local_cookies_login(self): + + if not os.path.exists(self.cookies_file): + raise LoginError('Local Cookies file not found') + + # cookies exists + util.log(f'Cookie file found: {self.cookies_file}, attempt to login with local cookie') + try: + with open(self.cookies_file, 'rb') as f: + local_cookies = pickle.load(f) + self._session.cookies = local_cookies + if self._check_is_logged(): + util.log('Logged in successfully with local cookies', inform=True) + return + else: + os.remove(self.cookies_file) + util.log('Removed outdated cookies', inform=True) + except pickle.UnpicklingError as e: + os.remove(self.cookies_file) + util.log('Removed corrupted cookies file, message: {}'.format(e)) + + # local cookies failed + raise LoginError('Login with cookies failed') + + def _user_cookies_login(self): + msg = 'Login with local cookies failed, would you like to provide a new cookies?' + os.linesep \ + + ' [y] Yesss!' + os.linesep \ + + ' [n] Noooo! (Attempt alternate login with username and password)' + util.log(msg, normal=True) + + while True: + answer = input(' [=] Please select an option:').strip().lower() + if answer in ['n', 'no']: + break + if answer not in ['y', 'yes']: + print('Please enter your answer as case-insensitive \'y\' or \'n\' or \'yes\' or \'no\'') + continue + + cookies = input(' [=] Please enter your cookies here, just php session id will work,' + os.linesep + + ' [=] e.g. PHPSESSIONID=1234567890:') + + try: + self._change_to_new_cookies(cookies) + if self._check_is_logged(): + self._save_cookies() + return + else: + util.log('Failed login with cookies entered, would you like to try again? [y/n]', normal=True) + except LoginError as e: + util.log(f'cookies entered is invalid: {e}') + util.log('would you like to try again? [y/n]') + + # user enter cookies failed + raise LoginError('Failed login with user cookies') + + def _change_to_new_cookies(self, user_cookies): + # remove old cookies + for old_cookie in self._session.cookies.keys(): + self._session.cookies.__delitem__(old_cookie) + + # add new cookies + try: + for new_cookie in user_cookies.split(';'): + name, value = new_cookie.split('=', 1) + self._session.cookies[name] = value + except ValueError as e: + raise LoginError(f'Cookies given is invalid, please try again | {e}') from e + + +class BookmarkHandler: + _bookmark_url = 'https://www.pixiv.net/ajax/user/{user_id}/illusts/bookmarks?' + + @classmethod + def _check_params(cls, limit, bookmark_type, restrict): + if limit and not isinstance(limit, int): + raise APIUserError(f'bookmark limit is not int or None') + if bookmark_type and not params.BookmarkType.is_valid(bookmark_type): + raise APIUserError(f'Invalid bookmark type: {bookmark_type}, must be type of {params.BookmarkType}') + if restrict and not params.Restrict.is_valid(restrict): + raise APIUserError(f'Invalid restrict: {restrict}, must be type of {params.Restrict}') + + @classmethod + def _set_params(cls, bookmark_type, restrict, user_id): + req_params = dict() + req_params['limit'] = 1 + req_params['offset'] = 0 + req_params['tag'] = '' + if bookmark_type: + if bookmark_type is params.BookmarkType.ILLUST_OR_MANGA: + pass # this is the default + + if restrict: + if restrict is params.Restrict.PUBLIC: + req_params['rest'] = 'show' + elif restrict is params.Restrict.PRIVATE: + req_params['rest'] = 'hide' + + return req_params + + @classmethod + def bookmarks(cls, limit, bookmark_type, restrict, user_id, session): + cls._check_params(limit, bookmark_type, restrict) + + req_params = cls._set_params(bookmark_type, restrict, user_id) + + try: + url = cls._bookmark_url.format(user_id=user_id) + data = util.req(url=url, params=req_params, session=session).json() + req_params['limit'] = data['body']['total'] + data = util.req(url=url, params=req_params, session=session).json() + ids = [item['id'] for item in data['body']['works']] + return util.trim_to_limit(ids, limit=limit) + except (ReqException, KeyError) as e: + raise APIUserError('Failed to retrieve bookmark') from e + + +class CreationHandler: + _illusts_url = 'https://www.pixiv.net/touch/ajax/illust/user_illusts?user_id={user_id}' + _content_url = 'https://www.pixiv.net/touch/ajax/user/illusts?' + + @classmethod + def illusts(cls, user_id, session, limit): + try: + res = util.req(session=session, url=cls._illusts_url.format(user_id=user_id)) + illust_ids = eval(res.text) # string to list + return util.trim_to_limit(illust_ids, limit) + except ReqException as e: + raise APIUserError(f'Failed to get illustration from user id: {user_id}') from e + + @classmethod + def mangas(cls, user_id, session, limit): + req_params = dict() + req_params['id'] = user_id + req_params['type'] = 'manga' + curr_page = 0 + last_page = 1 # a number more than curr_page + manga_ids = [] + + try: + while curr_page < last_page: + curr_page += 1 + + req_params['p'] = curr_page + data = util.req(session=session, url=cls._content_url, params=req_params).json() + manga_ids += [illust['id'] for illust in data['illusts']] + if limit: + if len(manga_ids) > limit: + manga_ids = util.trim_to_limit(items=manga_ids, limit=limit) + break + last_page = data['lastPage'] + except (ReqException, KeyError) as e: + raise APIUserError(f'Failed to get manga from user id: {user_id}') from e + + return manga_ids + + +class WebAPIUser(DefaultAPIUser): + + def bookmarks(self, limit: int = None, bookmark_type: params.Type = params.Type.ILLUST, + restrict: params.Restrict = params.Restrict.PUBLIC) -> List[int]: + return BookmarkHandler.bookmarks(bookmark_type=bookmark_type, restrict=restrict, limit=limit, + session=self._session, user_id=self.id) + + +class WebAPIClient(AccountClient, CookiesClient, DefaultAPIClient): + _self_details_url = 'https://www.pixiv.net/touch/ajax/user/self/status' + + def __init__(self, username, password): + super().__init__() + try: + AccountClient._login(self, username, password) + except LoginError: + try: + CookiesClient._login(self) + except LoginError: + raise LoginError('Web client login failed') + + DefaultAPIClient.__init__(self._session) + self._config() + + def _config(self): + try: + data = util.req(url=self._self_details_url, session=self._session).json() + self._name = data['body']['user_status']['user_name'] + self._id = data['body']['user_status']['user_id'] + self._account = data['body']['user_status']['user_account'] + self.user = self.visits(self.id) + except (ReqException, KeyError) as e: + raise APIUserError(f'Failed to configure self') from e + + def bookmarks(self, limit: int = None, bookmark_type: params.BookmarkType = params.BookmarkType.ILLUST_OR_MANGA, + restrict: params.Restrict = params.Restrict.PUBLIC) -> List[int]: + return BookmarkHandler.bookmarks(limit=limit, bookmark_type=bookmark_type, restrict=restrict, user_id=self.id, + session=self._session) + + def illusts(self, limit: int = None) -> List[int]: + return CreationHandler.illusts(user_id=self.id, limit=limit, session=self._session) + + def mangas(self, limit: int = None) -> List[int]: + return CreationHandler.mangas(user_id=self.id, session=self._session, limit=limit) + + def search(self, keyword: str = '', search_type: params.SearchType = params.SearchType.ILLUST_OR_MANGA, + match: params.Match = params.Match.PARTIAL, sort: params.Sort = params.Sort.DATE_DESC, + search_range: Union[datetime.timedelta, params.Range] = None, limit: int = None) -> List[int]: + return super().search(keyword=keyword, search_type=search_type, match=match, sort=sort, + search_range=search_range, limit=limit) + + def rank(self, limit: int = None, date: Union[str, datetime.date] = format(datetime.date.today(), '%Y%m%d'), + content: params.Content = params.Content.ILLUST, rank_type: params.RankType = params.RankType.DAILY) -> \ + List[int]: + return super().rank(rank_type=rank_type, date=date, content=content, limit=limit) + + def visits(self, user_id: int): + return WebAPIUser(user_id=user_id, session=self._session) + + @property + def account(self): + return self._account + + @property + def name(self): + return self._name + + @property + def id(self): + return self._id + + +def test(): + print('Testing Web Client') + from .. import settings + client = WebAPIClient(settings.username, settings.password) + ids = client.search(keyword='arknights', limit=234, sort=params.Sort.DATE_DESC, + search_type=params.SearchType.ILLUST_OR_MANGA, + match=params.Match.EXACT, + search_range=params.Range.A_MONTH) + assert len(ids) == 234, len(ids) + + ids = client.bookmarks(bookmark_type=params.BookmarkType.ILLUST_OR_MANGA, limit=34) + assert len(ids) == 34, len(ids) + + ids = client.illusts() + assert len(ids) == 0, len(ids) + + ids = client.mangas() + assert len(ids) == 0, len(ids) + + ids = client.rank(rank_type=params.RankType.ROOKIE, date=datetime.date.today(), content=params.Content.MANGA, + limit=50) + assert len(ids) == 50, len(ids) + + user_id = 38088 + user = client.visits(user_id=user_id) + user_illust_ids = user.illusts(limit=100) + assert len(user_illust_ids) == 100, len(user_illust_ids) + + user_manga_ids = user.mangas(limit=2) + assert len(user_manga_ids) == 2, len(user_manga_ids) + + print('Successfully tested web client') + + +def main(): + test() + + +if __name__ == '__main__': + main() diff --git a/gui/lib/pikax/downloader.py b/gui/lib/pikax/downloader.py new file mode 100644 index 0000000..7c6d03a --- /dev/null +++ b/gui/lib/pikax/downloader.py @@ -0,0 +1,76 @@ +import os +import re +from typing import Tuple, Iterator + +import requests + +from .models import BaseDownloader +from . import util +from .api.models import Artwork + + +class DefaultDownloader(BaseDownloader): + @staticmethod + def download_illust(artwork: Artwork, folder: str = '') -> Iterator[Tuple[Artwork.DownloadStatus, str]]: + artwork_detail = 'None' + folder = str(folder) + if folder and not os.path.isdir(folder): + os.mkdir(folder) + try: + for status, url_and_headers, filename in artwork: + url, headers = url_and_headers + page_num_search = re.search(r'\d{8}_p(\d*)', url) + page_num = page_num_search.group(1) if page_num_search else -1 + filename = os.path.join(util.clean_filename(str(folder)), util.clean_filename(str(filename))) + artwork_detail = f'[{str(artwork.title)}] p{page_num} by [{str(artwork.author)}]' + if status is Artwork.DownloadStatus.OK: + + if os.path.isfile(filename): + yield Artwork.DownloadStatus.SKIPPED, artwork_detail + continue + + with requests.get(url=url, headers=headers) as r: + r.raise_for_status() + with open(filename, 'wb') as file: + for chunk in r.iter_content(chunk_size=1024): + file.write(chunk) + yield Artwork.DownloadStatus.OK, artwork_detail + + except requests.RequestException as e: + yield Artwork.DownloadStatus.FAILED, artwork_detail + f': {e}' + + @staticmethod + def download_manga(artwork: Artwork, folder: str = None) -> Iterator[Tuple[Artwork.DownloadStatus, str]]: + return DefaultDownloader.download_illust(artwork=artwork, folder=folder) + + +def test(): + from . import settings + from .api.androidclient import AndroidAPIClient + from pikax.processor import DefaultIDProcessor + from . import params + from .models import PikaxResult + import shutil + + client = AndroidAPIClient(settings.username, settings.password) + processor = DefaultIDProcessor() + downloader = DefaultDownloader() + num_of_artworks = 53 + ids = client.rank(limit=num_of_artworks) + assert len(ids) == num_of_artworks, len(ids) + artworks, fails = processor.process(ids, process_type=params.ProcessType.ILLUST) + assert len(artworks) == num_of_artworks, len(artworks) + result = PikaxResult(download_type=params.DownloadType.ILLUST, artworks=artworks) + downloader.download(pikax_result=result, folder=settings.TEST_FOLDER) + shutil.rmtree(settings.TEST_FOLDER) + print(f'Removed test folder: {settings.TEST_FOLDER}') + + print('Successfully tested downloader') + + +def main(): + test() + + +if __name__ == '__main__': + main() diff --git a/gui/lib/pikax/exceptions.py b/gui/lib/pikax/exceptions.py new file mode 100644 index 0000000..43dcd28 --- /dev/null +++ b/gui/lib/pikax/exceptions.py @@ -0,0 +1,98 @@ +""" +This module contains different exceptions for Pikax +""" + + +class PikaxException(Exception): + """Base Exception for all exceptions in Pikax + """ + pass + + +class ReqException(PikaxException): + """ + This exception is raised when something when wrong with sending requests to servers + + **See Also** + :func: util.req + """ + pass + + +class PostKeyError(PikaxException): + """When failed to retrieve post key during login + + This may be a direct cause of LoginError + + **See Also** + :func: pages.LoginPage.login + :class: LoginError + """ + pass + + +class LoginError(PikaxException): + """When attempt to login has failed + + **See Also** + :func: pages.LoginPage.login + """ + pass + + +class ArtworkError(PikaxException): + """When failed to initialize a artwork object + + :func: items.Artwork.__init__ + """ + pass + + +class UserError(PikaxException): + """When failed to initialize a User object + + **See Also** + :func: items.User.__init__ + """ + pass + + +class SearchError(PikaxException): + """When failed to initialize a search in SearchPage + + **See Also** + :func: pages.SearchPage.search + """ + pass + + +class RankError(PikaxException): + pass + + +class BaseClientException(PikaxException): + pass + + +class ClientException(BaseClientException): + pass + + +class ProcessError(PikaxException): + pass + + +class PikaxResultError(PikaxException): + pass + + +class ParamsException(PikaxException): + pass + + +class PikaxUserError(PikaxException): + pass + + +class APIUserError(PikaxException): + pass diff --git a/gui/lib/pikax/items.py b/gui/lib/pikax/items.py new file mode 100644 index 0000000..ad8aac3 --- /dev/null +++ b/gui/lib/pikax/items.py @@ -0,0 +1,85 @@ +import enum +import os + +from .exceptions import LoginError +from .api.androidclient import AndroidAPIClient +from .api.webclient import WebAPIClient +from .api.defaultclient import DefaultAPIClient +from . import util + + +class LoginHandler: + class LoginStatus(enum.Enum): + PC = enum.auto() + ANDROID = enum.auto() + LOG_OUT = enum.auto() + + def __init__(self, username=None, password=None): + self.username = username + self.password = password + + def web_login(self, username=None, password=None): + if username and password: + self.username = username + self.password = password + + try: + util.log('Attempting Web Login ...') + return self.LoginStatus.PC, WebAPIClient(self.username, self.password) + except LoginError as e: + util.log(f'web login failed: {e}') + return self.LoginStatus.LOG_OUT, DefaultAPIClient() + + def android_login(self, username=None, password=None): + if username and password: + self.username = username + self.password = password + + try: + util.log('Attempting Android Login ...') + return self.LoginStatus.ANDROID, AndroidAPIClient(self.username, self.password) + except LoginError as e: + util.log(f'android login failed: {e}') + return self.LoginStatus.LOG_OUT, DefaultAPIClient() + + def login(self, username=None, password=None): + if username and password: + self.username = username + self.password = password + + try: + util.log('Attempting Web Login ...') + + client = WebAPIClient(self.username, self.password) + login_status = self.LoginStatus.PC + + except LoginError as e: + util.log(f'Web Login failed: {e}') + try: + util.log('Attempting Android Login ...') + + client = AndroidAPIClient(self.username, self.password) + login_status = self.LoginStatus.ANDROID + + except LoginError as e: + util.log(f'Android Login failed: {e}') + + client = DefaultAPIClient() + login_status = self.LoginStatus.LOG_OUT + + util.log(f'Login Status: {login_status}, API Client: {client}', start=os.linesep, inform=True) + return login_status, client + + +def main(): + from . import settings + loginer = LoginHandler(settings.username, settings.password) + status, client = loginer.web_login() + assert status is LoginHandler.LoginStatus.PC + status, client = loginer.android_login() + assert status is LoginHandler.LoginStatus.ANDROID + + print('Successfully tested login handler') + +if __name__ == '__main__': + main() diff --git a/gui/lib/pikax/models.py b/gui/lib/pikax/models.py new file mode 100644 index 0000000..e268871 --- /dev/null +++ b/gui/lib/pikax/models.py @@ -0,0 +1,407 @@ +import datetime +import functools +import math +import operator +import os +from multiprocessing.dummy import Pool +from typing import Union, List, Tuple, Iterator + +from . import params, settings, util +from .api.models import Artwork + + +class PikaxResult: + """ + This is the interface for result return by operation such as search, rank or result.likes > 1000 etc... + """ + + def __init__(self, artworks: List[Artwork], download_type: params.DownloadType, folder: str = ''): + if any(not issubclass(artwork.__class__, Artwork) for artwork in artworks): + from .exceptions import PikaxResultError + raise PikaxResultError(f'artworks must all be subclass of {Artwork}') + + self._artworks = artworks + self._folder = str(folder) + self._download_type = download_type + maker = functools.partial(self.result_maker, download_type=download_type) + self._likes = self.ComparableItem(self, maker, 'likes') + self._bookmarks = self.ComparableItem(self, maker, 'bookmarks') + self._views = self.ComparableItem(self, maker, 'views') + + def result_maker(self, artworks, download_type, folder): + raise NotImplemented + + def __add__(self, other: 'PikaxResult') -> 'PikaxResult': + """ + This provide implementation of + of PikaxResult which result in a PikaxResult + returned after adding artworks in both result + :param other: the other PikaxResult, they must have same DownloadType + :rtype: PikaxResult + """ + raise NotImplementedError + + def __sub__(self, other: 'PikaxResult') -> 'PikaxResult': + """ + This provide implementation of - of PikaxResult which result in a PikaxResult + contains all artworks in the self which are not in other + :rtype: PikaxResult + """ + raise NotImplementedError + + def __getitem__(self, index: int) -> Artwork: + raise NotImplementedError + + def __len__(self) -> int: + raise NotImplementedError + + @property + def artworks(self): + return self._artworks + + @property + def folder(self): + return self._folder + + class ComparableItem: + _operator_to_symbol = { + operator.eq: '==', + operator.ne: '!=', + operator.gt: '>', + operator.lt: '<', + operator.ge: '>=', + operator.le: '<=', + } + + _operator_to_name = { + operator.eq: 'eq', + operator.ne: 'ne', + operator.gt: 'gt', + operator.lt: 'lt', + operator.ge: 'ge', + operator.le: 'le', + } + + def __init__(self, outer_self, result_maker, name): + self.name = name + self.outer_self = outer_self + self.result_maker = result_maker + + def __eq__(self, value): + return self._compare(operator.eq, value) + + def __ne__(self, value): + return self._compare(operator.ne, value) + + def __gt__(self, value): + return self._compare(operator.gt, value) + + def __ge__(self, value): + return self._compare(operator.ge, value) + + def __lt__(self, value): + return self._compare(operator.lt, value) + + def __le__(self, value): + return self._compare(operator.le, value) + + def _compare(self, compare_operator, value): + operator_symbol = self._operator_to_symbol[compare_operator] + util.log(f'Filtering {self.name} {operator_symbol} {value}', start=os.linesep, inform=True) + + old_len = len(self.outer_self.artworks) + new_artworks = list( + filter(lambda item: compare_operator(getattr(item, self.name), value), self.outer_self.artworks)) + new_len = len(new_artworks) + + operator_name = self._operator_to_name[compare_operator] + folder = util.clean_filename(str(self.outer_self.folder) + '_' + operator_name + '_' + str(value)) + result = self.result_maker(artworks=new_artworks, folder=folder) + + util.log(f'[ done ] {old_len} => {new_len}', inform=True) + return result + + @property + def likes(self) -> ComparableItem: + return self._likes + + @property + def views(self) -> ComparableItem: + return self._views + + @property + def bookmarks(self) -> ComparableItem: + return self._bookmarks + + @property + def download_type(self) -> params.DownloadType: + return self._download_type + + +class PikaxUserInterface: + """ + This is the interface of user returned in Pikax operation such as Pikax.visits + """ + + def illusts(self, limit: int = None) -> PikaxResult: + """ + Return the illustrations uploaded by this user on Pixiv + :param limit: Number of illustrations to return + :rtype: PikaxResult + """ + raise NotImplementedError + + def mangas(self, limit: int = None) -> PikaxResult: + """ + Return the mangas uploaded by this user on Pixiv + :param limit: Number of mangas to return + :rtype: PikaxResult + """ + raise NotImplementedError + + def bookmarks(self, limit: int = None, + bookmark_type: params.BookmarkType = params.BookmarkType.ILLUST_OR_MANGA) -> PikaxResult: + """ + Return the bookmarks saved by this user on Pixiv + :param bookmark_type: The type of bookmark to return, must be enum of params.BookmarkType + :param limit: Number of mangas to return + :rtype: PikaxResult + """ + raise NotImplementedError + + @property + def id(self) -> int: + """ + The id of this user in Pixiv + :rtype: int + """ + raise NotImplementedError + + @property + def name(self) -> str: + """ + The user name of this user on Pixiv + :rtype: str + """ + raise NotImplementedError + + @property + def account(self) -> str: + """ + The account name of this user on Pixiv + :rtype: str + """ + raise NotImplementedError + + +class PikaxPagesInterface: + """ + The methods to implement if the subclass support pages operations + """ + + def search(self, keyword: str = '', + search_type: params.Type = params.Type.ILLUST, + match: params.Match = params.Match.PARTIAL, + sort: params.Sort = None, + search_range: datetime.timedelta = None, + popularity: int = None, + limit: int = None) \ + -> PikaxResult: + """ + + Perform search on Pixiv and returns the results + + :param keyword: the word to search + :param search_type: type of artwork to search + :param match: define how strict the keywords are matched against artworks + :param sort: order of the search result + :param search_range: the date offset from today, can be a datetime.timedelta object + :param popularity: if given, {popularity}users入り will be added after keywords + :param limit: return number of artwork specified by limit, all by default + :return: an object implement PikaxResult + :rtype: PikaxResult + """ + raise NotImplementedError + + def rank(self, + limit: int = None, + date: Union[str, datetime.datetime] = format(datetime.datetime.today(), '%Y%m%d'), + content: params.Content = params.Content.ILLUST, + rank_type: params.RankType = params.RankType.DAILY) \ + -> PikaxResult: + """ + + Retrieve ranking's artworks from Pixiv + + :param limit: the number of artworks to return + :param date: the date of ranking + :param content: the type of artwork to rank + :param rank_type: the mode for ranking, daily, monthly etc ... + :return: an object implement PikaxResult + :rtype: PikaxResult + """ + raise NotImplementedError + + +class PikaxInterface(PikaxPagesInterface): + """ + The api entry interface + """ + + def search(self, keyword: str = '', search_type: params.SearchType = params.SearchType.ILLUST_OR_MANGA, + match: params.Match = params.Match.PARTIAL, sort: params.Sort = None, popularity: int = None, + search_range: datetime.timedelta = None, limit: int = None) -> PikaxResult: + """ + + Perform search on Pixiv and returns the results + + :param keyword: the word to search + :param search_type: type of artwork to search + :param match: define how strict the keywords are matched against artworks + :param sort: order of the search result + :param search_range: the date offset from today, can be a datetime.timedelta object + :param popularity: if given, {popularity}users入り will be added after keywords + :param limit: return number of artwork specified by limit, all by default + :return: an object implement PikaxResult + :rtype: PikaxResult + """ + raise NotImplementedError + + def rank(self, limit: int = None, date: Union[str, datetime.datetime] = format(datetime.datetime.today(), '%Y%m%d'), + content: params.Content = params.Content.ILLUST, + rank_type: params.RankType = params.RankType.DAILY) -> PikaxResult: + """ + + Retrieve ranking's artworks from Pixiv + + :param limit: the number of artworks to return + :param date: the date of ranking + :param content: the type of artwork to rank + :param rank_type: the mode for ranking, daily, monthly etc ... + :return: an object implement PikaxResult + :rtype: PikaxResult + """ + raise NotImplementedError + + def login(self, username: str = settings.username, password: str = settings.password) \ + -> (PikaxUserInterface, PikaxPagesInterface): + raise NotImplementedError + + def download(self, pikax_result: PikaxResult, folder: str = None) -> None: + """ + Download all items given + + :param pikax_result: a PikaxResult to download, default None + :param folder: the folder where artworks are download, default using folder in settings.py + :param illust_id: the illust id to download, default None + :rtype: None + """ + raise NotImplementedError + + def visits(self, user_id: int) -> PikaxUserInterface: + """ + Access a user in Pixiv given the user id with best available client + + :param user_id: the user id of the member in Pixiv + :return: an object implement PikaxUserInterface + :rtype: PikaxUserInterface + """ + raise NotImplementedError + + +class BaseDownloader: + + @staticmethod + def download_illust(artwork: Artwork, folder: str = None) -> Iterator[Tuple[Artwork.DownloadStatus, str]]: + raise NotImplementedError + + @staticmethod + def download_manga(artwork: Artwork, folder: str = None) -> Iterator[Tuple[Artwork.DownloadStatus, str]]: + raise NotImplementedError + + def __init__(self): + self.download_type_to_function = { + params.DownloadType.ILLUST: self.download_illust, + params.DownloadType.MANGA: self.download_manga, + } + + # @staticmethod + # def config_artworks(artworks: List[Artwork]): + # util.log('Configuring artworks', start=os.linesep, inform=True) + # total = len(artworks) + # config_artworks = [] + # failed_config_artworks = dict() # reason map to artwork + # pool = Pool() + # + # def config_artwork(artwork_item): + # try: + # artwork_item.config() + # config_artworks.append(artwork_item) + # except ArtworkError as e: + # failed_config_artworks[str(e)] = artwork_item + # + # for index, _ in enumerate(pool.imap_unordered(config_artwork, artworks)): + # util.print_progress(index + 1, total) + # + # msg = f'expected: {total} | success: {len(config_artworks)} | failed: {len(failed_config_artworks)}' + # util.print_done(msg) + # + # if failed_config_artworks: + # for index, item in enumerate(failed_config_artworks.items()): + # util.log(f'Artwork with id: {item[1].id} failed config for download: {item[0]}', error=True) + # + # return config_artworks + + def download(self, pikax_result: PikaxResult, folder: str = ''): + + if not folder: + folder = pikax_result.folder + + folder = util.clean_filename(folder) + + if folder and not os.path.isdir(folder): + os.mkdir(folder) + + download_function = self.download_type_to_function[pikax_result.download_type] + download_function = functools.partial(download_function, folder=folder) + artworks = pikax_result.artworks + successes = [] + fails = [] + skips = [] + total_pages = sum(len(artwork) for artwork in artworks) + total_artworks = len(artworks) + curr_page = 0 + curr_artwork = 0 + pool = Pool() + util.log(f'Downloading Artworks | {total_pages} pages from {total_artworks} artworks', start=os.linesep, + inform=True) + + for download_details in pool.imap_unordered(download_function, artworks): + curr_artwork += 1 + for download_detail in download_details: + curr_page += 1 + if settings.MAX_PAGES_PER_ARTWORK and curr_page > settings.MAX_PAGES_PER_ARTWORK: + break + status, msg = download_detail + info = str(msg) + ' ' + str(status.value) + if status is Artwork.DownloadStatus.OK: + successes.append(msg) + elif status is Artwork.DownloadStatus.SKIPPED: + skips.append(msg) + else: + fails.append(msg) + info = f'{curr_artwork} / {total_artworks} ' \ + f'=> {math.ceil((curr_artwork / total_artworks) * 100)}% | ' + info + yield curr_page, total_pages, info + util.print_done() + + util.log(f'There are {len(successes)} downloaded pages', inform=True) + + util.log(f'There are {len(skips)} skipped pages', inform=True) + for index, skip_info in enumerate(skips): + util.log(skip_info, start=f' [{index + 1}] ', inform=True) + + util.log(f'There are {len(fails)} failed pages', inform=True) + for index, skip_info in enumerate(fails): + util.log(skip_info, start=f' [{index + 1}] ', inform=True) + + util.print_done(str(folder)) diff --git a/gui/lib/pikax/params.py b/gui/lib/pikax/params.py new file mode 100644 index 0000000..6a50e68 --- /dev/null +++ b/gui/lib/pikax/params.py @@ -0,0 +1,173 @@ +import calendar +import datetime +import enum + + +class PikaxEnum(enum.Enum): + @classmethod + def is_valid(cls, value): + return isinstance(value, cls) + + +class Type(PikaxEnum): + ILLUST = 'illust' + USER = 'user' + MANGA = 'manga' + + _member_to_container_map = { + 'illust': 'illusts', + 'user': 'user_previews', + 'manga': 'illusts', # intended + } + + @classmethod + def get_response_container_name(cls, key): + return cls._member_to_container_map.value[key] + + +class Match(PikaxEnum): + # illusts and novel match + EXACT = 'exact_match_for_tags' + PARTIAL = 'partial_match_for_tags' + # illusts only + ANY = 'title_and_caption' + + +class Sort(PikaxEnum): + DATE_DESC = 'date_desc' + DATE_ASC = 'date_asc' + + +class RankType(PikaxEnum): + DAILY = 'daily' + WEEKLY = 'weekly' + MONTHLY = 'monthly' + ROOKIE = 'rookie' + + +class Dimension(PikaxEnum): + HORIZONTAL = '0.5' + VERTICAL = '-0.5' + SQUARE = '0' + + +class Range(PikaxEnum): + A_DAY = datetime.timedelta(days=1) + A_WEEK = datetime.timedelta(days=7) + A_MONTH = datetime.timedelta( + days=calendar.monthrange(year=datetime.date.today().year, month=datetime.date.today().month)[1]) + A_YEAR = datetime.timedelta(days=365 + calendar.isleap(datetime.date.today().year)) + + +class Date(PikaxEnum): + TODAY = format(datetime.date.today(), '%Y%m%d') + + +# collections params, e.g. illusts, novels +class Restrict(PikaxEnum): + PUBLIC = 'public' + PRIVATE = 'private' + + +class CreationType(PikaxEnum): + ILLUST = 'illust' + MANGA = 'manga' + + +class DownloadType(PikaxEnum): + ILLUST = enum.auto() + MANGA = enum.auto() + + +class ProcessType(PikaxEnum): + ILLUST = 'illust' + MANGA = 'manga' + + _process_to_download_map = { + ILLUST: DownloadType.ILLUST, + MANGA: DownloadType.MANGA + } + + @classmethod + def map_process_to_download(cls, process_type): + if cls.is_valid(process_type): + return cls._process_to_download_map.value[process_type.value] + else: + raise KeyError(f'process type: {process_type} is not type of {ProcessType}') + + +class SearchType(PikaxEnum): + ILLUST_OR_MANGA = 'illust' + # USER = 'user' # XXX: Need implementation + + _search_to_process_map = { + ILLUST_OR_MANGA: ProcessType.ILLUST + } + + @classmethod + def map_search_to_process(cls, search_type): + if cls.is_valid(search_type): + return cls._search_to_process_map.value[search_type.value] + else: + raise KeyError(f'search type: {search_type} is not type of {SearchType}') + + +class Content(PikaxEnum): + ILLUST = 'illust' + MANGA = 'manga' + + _content_to_process_map = { + ILLUST: ProcessType.ILLUST, + MANGA: ProcessType.MANGA + } + + @classmethod + def map_content_to_process(cls, content_type): + if cls.is_valid(content_type): + return cls._content_to_process_map.value[content_type.value] + else: + raise KeyError(f'content type: {content_type} is not type of {Content}') + + +class BookmarkType(PikaxEnum): + ILLUST_OR_MANGA = 'illust' + + _bookmark_to_process_map = { + 'illust': ProcessType.ILLUST + } + + _bookmark_to_download_map = { + 'illust': DownloadType.ILLUST + } + + @classmethod + def map_bookmark_to_process(cls, bookmark_type): + if cls.is_valid(bookmark_type): + return cls._bookmark_to_process_map.value[bookmark_type.value] + else: + raise KeyError(f'bookmark type: {bookmark_type} is not type of {cls}') + + @classmethod + def map_bookmark_to_download(cls, bookmark_type): + if cls.is_valid(bookmark_type): + return cls._bookmark_to_download_map.value[bookmark_type.value] + else: + raise KeyError(f'bookmark type: {bookmark_type} is not type of {cls}') + + +# for testing +def main(): + assert Type.is_valid(Date.DATE_DESC) is False + assert Type.is_valid(Date.DATE_ASC) is False + assert Type.is_valid(Match.ANY) is False + assert Type.is_valid(Match.EXACT) is False + assert Type.is_valid(Match.PARTIAL) is False + assert Type.is_valid(Match.TEXT) is False + assert Type.is_valid(Match.KEYWORD) is False + assert Type.is_valid(Type.ILLUST) + assert Type.is_valid(Type.NOVEL) + assert Type.is_valid(Type.USER) + + +if __name__ == '__main__': + main() diff --git a/gui/lib/pikax/pikax.py b/gui/lib/pikax/pikax.py new file mode 100644 index 0000000..e2499a8 --- /dev/null +++ b/gui/lib/pikax/pikax.py @@ -0,0 +1,237 @@ +import datetime +from typing import Union + +from . import params, settings, util +from .api.artwork import Illust +from .api.defaultclient import DefaultAPIClient +from .downloader import DefaultDownloader +from .items import LoginHandler +from .models import PikaxInterface, PikaxUserInterface, PikaxResult, PikaxPagesInterface +from .processor import DefaultIDProcessor +from .result import DefaultPikaxResult +from .user import DefaultPikaxUser + + +class Pikax(PikaxInterface): + """ + The entry point of this api + """ + + def __init__(self, username=None, password=None): + self._login_handler = LoginHandler() + self.default_client = DefaultAPIClient() + self.id_processor = DefaultIDProcessor() + self.downloader = DefaultDownloader() + self.username = password + self.password = username + self.web_client = None + self.android_client = None + + if username and password: + self.login() + + def login(self, username: str = settings.username, password: str = settings.password) -> ( + PikaxUserInterface, PikaxPagesInterface): + """ + Attempt login using web and android method and returns the logged user + if succeed, else returns None + This method also saves logged client which used to perform other actions + + :param username: Username for login + :param password: Password for login + :return: a logged PikaxUser implemented PikaxUserInterface or None if failed + :rtype: PikaxUserInterface or None + """ + + util.log('Attempting Login', inform=True) + + if username and password: + self.username = username + self.password = password + + status, client = self._login_handler.web_login(self.username, self.password) + if status is LoginHandler.LoginStatus.PC: + self.web_client = client + util.log('successfully logged in as web user', inform=True) + + status, client = self._login_handler.android_login(self.username, self.password) + if status is LoginHandler.LoginStatus.ANDROID: + self.android_client = client + util.log('successfully logged in as android user', inform=True) + + if not (self.android_client or self.web_client): + util.log('failed login, using default client, some features will be unavailable', inform=True) + return None + + logged_client = self._get_client() + return DefaultPikaxUser(client=logged_client, user_id=logged_client.id) + + def search(self, keyword: str = '', + search_type: params.SearchType = params.SearchType.ILLUST_OR_MANGA, + match: params.Match = params.Match.PARTIAL, + sort: params.Sort = params.Sort.DATE_DESC, + search_range: datetime.timedelta = None, + popularity: int = None, + limit: int = None) \ + -> PikaxResult: + """ + Search pixiv with best available client. + Note that pixiv returns less result if not logged in + + :param keyword: the word to search + :param search_type: type of artwork to search + :param match: define how strict the keywords are matched against artworks + :param sort: order of the search result + :param search_range: the date offset from today, can be a datetime.timedelta object + :param popularity: if given, {popularity} users入り will be added after keywords + :param limit: return number of artwork specified by limit, all by default + :return: an object implement PikaxResult + :rtype: PikaxResult + """ + + util.log(f'Searching {keyword} of type {search_type} with limit {limit}', inform=True) + + client = self._get_client() + if popularity: + keyword = self._add_popularity_to_keyword(keyword, popularity) + ids = client.search(keyword=keyword, search_type=search_type, match=match, sort=sort, + search_range=search_range, limit=limit) + util.print_done(f'number of ids: {len(ids)}') + process_type = self._get_process_from_search(search_type) + download_type = self._get_download_from_process(process_type) + success, fail = self._get_id_processor().process(ids, process_type=process_type) + folder = settings.DEFAULT_SEARCH_FOLDER.format(keyword=keyword, search_type=search_type, match=match, sort=sort, + search_range=search_range, popularity=popularity, limit=limit) + return DefaultPikaxResult(success, download_type=download_type, folder=folder) + + def rank(self, limit: int = datetime.datetime, + date: Union[str, datetime.datetime] = format(datetime.datetime.today(), '%Y%m%d'), + content: params.Content = params.Content.ILLUST, + rank_type: params.RankType = params.RankType.DAILY) \ + -> PikaxResult: + """ + Return the ranking artworks in pixiv according to parameters. + This method returns complete artworks even if not logged in + + :param limit: the number of artworks to return + :param date: the date of ranking + :param content: the type of artwork to rank + :param rank_type: the mode for ranking, daily, monthly etc ... + :return: an object implement PikaxResult + :rtype: PikaxResult + """ + + util.log(f'Ranking date {date} of type {rank_type} and content {content} with limit {limit}', inform=True) + + client = self._get_client() + ids = client.rank(rank_type=rank_type, content=content, date=date, limit=limit) + util.print_done(f'number of ids: {len(ids)}') + process_type = self._get_process_from_content(content) + download_type = self._get_download_from_process(process_type) + success, fail = self._get_id_processor().process(ids, process_type=process_type) + folder = settings.DEFAULT_RANK_FOLDER.format(limit=limit, date=date, content=content, rank_type=rank_type) + return DefaultPikaxResult(success, download_type=download_type, folder=folder) + + def download(self, pikax_result=None, folder: str = '') -> None: + """ + Download all items given + + :param pikax_result: a PikaxResult to download, default None + :param folder: the folder where artworks are download, default using folder in settings.py + :param illust_id: the illust id to download, default None + :rtype: None + """ + for curr, total, info in self.downloader.download(pikax_result=pikax_result, folder=folder): + yield curr, total, info + + def visits(self, user_id: int) -> PikaxUserInterface: + """ + Access a user in Pixiv given the user id with best available client + + :param user_id: the user id of the member in Pixiv + :return: an object implement PikaxUserInterface + :rtype: PikaxUserInterface + """ + client = self._get_client() + return DefaultPikaxUser(client=client, user_id=user_id) + + def _get_client(self): + if self.web_client: + return self.web_client + elif self.android_client: + return self.android_client + else: + return self.default_client + + def _get_id_processor(self): + return self.id_processor + + @staticmethod + def _add_popularity_to_keyword(keyword, popularity): + return str(keyword) + f' {popularity}users入り' + + @staticmethod + def _get_process_from_search(search_type): + return params.SearchType.map_search_to_process( + search_type) + + @staticmethod + def _get_download_from_process(process_type): + return params.ProcessType.map_process_to_download( + process_type) + + @staticmethod + def _get_process_from_content(content): + return params.Content.map_content_to_process(content) + + +def test(): + from . import settings + import shutil + pikax = Pikax(settings.username, settings.password) + + result = pikax.search(keyword='arknights', limit=15) + test_folder = settings.TEST_FOLDER + pikax.download(result, folder=test_folder) + + result = pikax.rank(limit=25) + pikax.download(result, folder=test_folder) + + user = pikax.login(settings.username, settings.password) + illusts = user.illusts() + assert len(illusts) == 0, len(illusts) + + bookmarks = user.bookmarks(limit=30) + assert len(bookmarks) == 30, len(bookmarks) + + mangas = user.mangas() + assert len(mangas) == 0, len(mangas) + + user = pikax.visits(user_id=1113943) + + ill = user.illusts(limit=25) + assert len(ill) == 25, len(ill) + + man = user.mangas(limit=10) + assert len(man) == 10, len(man) + + col = user.bookmarks(limit=10) + assert len(col) == 10, len(col) + + shutil.rmtree(test_folder) + print(f'removed test folder: {test_folder}') + + print('successfully tested pikax') + + +def main(): + # test() + from . import settings + pixiv = Pikax() + pixiv.login(settings.username, settings.password) + res = pixiv.search(keyword='arknights', popularity=1000, limit=10) + pixiv.download(res) + + +if __name__ == '__main__': + main() diff --git a/gui/lib/pikax/processor.py b/gui/lib/pikax/processor.py new file mode 100644 index 0000000..78d32e3 --- /dev/null +++ b/gui/lib/pikax/processor.py @@ -0,0 +1,44 @@ +from .api.artwork import Illust +from .api.models import BaseIDProcessor + + +class DefaultIDProcessor(BaseIDProcessor): + + def __init__(self): + super().__init__() + + def process_mangas(self, ids): + # they are essentially the same, just illust with more pages + return self.process_illusts(ids) + + def process_illusts(self, ids): + return self._general_processor(Illust, ids) + + +def test(): + from . import settings + from .items import LoginHandler + from . import params + import sys + sys.stdout.reconfigure(encoding='utf-8') + print('Testing Processor') + status, client = LoginHandler(settings.username, settings.password).login() + print(status, client) + processor = DefaultIDProcessor() + ids = client.rank(limit=100) + successes, failed = processor.process(ids, params.ProcessType.ILLUST) + assert len(successes) == 100, len(successes) + + ids = client.search(keyword='arknights', search_type=params.SearchType.ILLUST_OR_MANGA, limit=50) + successes, fails = processor.process(ids, params.ProcessType.ILLUST) + assert len(successes) == 50, len(successes) + + print('Successfully tested processor') + + +def main(): + test() + + +if __name__ == '__main__': + main() diff --git a/gui/lib/pikax/result.py b/gui/lib/pikax/result.py new file mode 100644 index 0000000..18b81b8 --- /dev/null +++ b/gui/lib/pikax/result.py @@ -0,0 +1,34 @@ +from .exceptions import PikaxResultError +from .api.models import Artwork +from .models import PikaxResult + + +class DefaultPikaxResult(PikaxResult): + + def __init__(self, artworks, download_type, folder=''): + super().__init__(artworks, download_type, folder) + + def result_maker(self, artworks, download_type, folder): + return DefaultPikaxResult(artworks, download_type, folder) + + def __add__(self, other: 'PikaxResult') -> 'PikaxResult': + if self._download_type is not other.download_type: + raise PikaxResultError( + f'PikaxResults are in different type: {self._download_type} and {other.download_type}') + new_artworks = list(set(self.artworks + other.artworks)) + new_folder = self.folder + '_added_to_' + other.folder + return DefaultPikaxResult(artworks=new_artworks, download_type=self._download_type, folder=new_folder) + + def __sub__(self, other: 'PikaxResult') -> 'PikaxResult': + if self._download_type is not other.download_type: + raise PikaxResultError( + f'PikaxResults are in different type: {self._download_type} and {other.download_type}') + new_artworks = [artwork for artwork in self.artworks if artwork not in other.artworks] + new_folder = self.folder + '_subbed_by_' + other.folder + return DefaultPikaxResult(new_artworks, download_type=self._download_type, folder=new_folder) + + def __getitem__(self, index: int) -> Artwork: + return self.artworks[index] + + def __len__(self) -> int: + return len(self.artworks) diff --git a/gui/lib/pikax/settings.py b/gui/lib/pikax/settings.py new file mode 100644 index 0000000..a1ebf7d --- /dev/null +++ b/gui/lib/pikax/settings.py @@ -0,0 +1,110 @@ +""" +default headers for sending requests +not all requests uses this headers +""" +DEFAULT_HEADERS = { + 'referer': 'https://www.pixiv.net/', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36' +} + +""" +default time out in seconds for requests +""" +TIMEOUT = 10 + +""" +Number of retries for requesting +""" +MAX_RETRIES_FOR_REQUEST = 3 + +""" +Minimum waiting time in seconds between two successive requests, +Set this number to a suitable amount to reduce +chances of getting block +""" +DELAY_PER_REQUEST = None + +""" +Proxies used for sending requests, +uses requests, map protocol to scheme, +# https://2.python-requests.org/en/master/user/advanced/ +e.g. +proxies = { + 'http': 'http://10.10.1.10:3128', + 'https': 'http://10.10.1.10:1080', +} +""" +REQUEST_PROXIES = {} + +""" +LOG_TYPE +'inform': print successive stage and error only +'std': allow debug printings +'save': save error to LOG_FILE only +""" +LOG_STD = False +LOG_INFORM = True +LOG_WARN = False +LOG_SAVE = False + +""" +file used to log error if save is included in LOG_TYPE +""" +LOG_FILE = 'log.txt' + +""" +default folder format for downloading, +do not change, reference only, +you can specify a new folder when calling pikax.download +""" +DEFAULT_MANGAS_FOLDER = '#{name}\'s mangas' +DEFAULT_ILLUSTS_FOLDER = '#{name}\'s illusts' +DEFAULT_BOOKMARKS_FOLDER = '#{name}\'s bookmarks' +DEFAULT_SEARCH_FOLDER = '#PixivSearch_{keyword}_{search_type}_{match}_{sort}_{search_range}_{popularity}_{limit}' +DEFAULT_RANK_FOLDER = '#PixivRanking_{date}_{rank_type}_{content}_{limit}' + +""" +String to clear previous stdout line +""" +CLEAR_LINE = '\r' + ' ' * 150 + '\r' + +""" +Indicate a failure when there's too much exceptions occurred during requesting in the same loop +""" +MAX_WHILE_TRUE_LOOP_EXCEPTIONS = 3 + +""" +Default request error message, +when error message is not given as param to util.req +""" +DEFAULT_REQUEST_ERROR_MSG = 'Exception while {type}' + +""" +A artwork id may have multiple pages, +sometimes is not desire to download all of them, +None means download all pages +""" +MAX_PAGES_PER_ARTWORK = None + +""" +file for saving cookies +""" +COOKIES_FILE = 'cookies.data' + +""" +default whether to log requests to stdout +""" +LOG_REQUEST = False + +""" +folder used when testing, do not run test if you are using folder of this name +or change this name before running test +""" +TEST_FOLDER = '#test_folder' + +""" +user name and password used to login +""" +username = 'restorecyclebin@gmail.com' +password = '123456' diff --git a/gui/lib/pikax/user.py b/gui/lib/pikax/user.py new file mode 100644 index 0000000..b5ffdc1 --- /dev/null +++ b/gui/lib/pikax/user.py @@ -0,0 +1,81 @@ +from .models import PikaxUserInterface +from . import params, settings +from .processor import DefaultIDProcessor +from .exceptions import PikaxUserError +from .result import DefaultPikaxResult, PikaxResult + + +class DefaultPikaxUser(PikaxUserInterface): + + def __init__(self, client, user_id): + self._client = client + self._user = self._client.visits(user_id=user_id) + self._id_processor = DefaultIDProcessor() + self._illusts_folder = settings.DEFAULT_ILLUSTS_FOLDER + self._mangas_folder = settings.DEFAULT_MANGAS_FOLDER + self._bookmarks_folder = settings.DEFAULT_BOOKMARKS_FOLDER + + def illusts(self, limit: int = None) -> PikaxResult: + ids = self._user.illusts(limit=limit) + successes, fails = self._id_processor.process(ids, process_type=params.ProcessType.ILLUST) + return DefaultPikaxResult(artworks=successes, download_type=params.DownloadType.ILLUST, + folder=self._illusts_folder.format(name=self.name)) + + def mangas(self, limit: int = None) -> PikaxResult: + ids = self._user.mangas(limit=limit) + successes, fails = self._id_processor.process(ids, process_type=params.ProcessType.MANGA) + return DefaultPikaxResult(artworks=successes, download_type=params.DownloadType.MANGA, + folder=self._mangas_folder.format(name=self.name)) + + def bookmarks(self, limit: int = None, + bookmark_type: params.BookmarkType = params.BookmarkType.ILLUST_OR_MANGA) -> PikaxResult: + try: + ids = self._user.bookmarks(limit=limit, bookmark_type=bookmark_type) + except NotImplementedError as e: + raise PikaxUserError('Failed to retrieve bookmark ids') from e + + try: + successes, fails = self._id_processor.process(ids, process_type=params.BookmarkType.map_bookmark_to_process( + bookmark_type)) + except NotImplementedError as e: + raise PikaxUserError('Failed to process bookmark id') from e + + return DefaultPikaxResult(artworks=successes, + download_type=params.BookmarkType.map_bookmark_to_download(bookmark_type), + folder=self._bookmarks_folder.format(name=self.name)) + + @property + def id(self): + return self._user.id + + @property + def account(self): + return self._user.account + + @property + def name(self): + return self._user.name + + +def test(): + from . import settings + from .api.androidclient import AndroidAPIClient + client = AndroidAPIClient(settings.username, settings.password) + user = DefaultPikaxUser(client, user_id=853087) + + result = user.illusts(limit=100) + assert len(result.artworks) == 100, len(result.artworks) + result = user.bookmarks(limit=100, bookmark_type=params.BookmarkType.ILLUST_OR_MANGA) + assert len(result.artworks) == 100, len(result.artworks) + result = user.mangas(limit=3) + assert len(result.artworks) == 3, len(result.artworks) + + print('Successfully tested user') + + +def main(): + test() + + +if __name__ == '__main__': + main() diff --git a/gui/lib/pikax/util.py b/gui/lib/pikax/util.py new file mode 100644 index 0000000..3c84cd5 --- /dev/null +++ b/gui/lib/pikax/util.py @@ -0,0 +1,339 @@ +""" +This module contains utilities/tools for pikax + +:func log: print according to parameters and settings +:func req: attempt to send network requests using requests lib and returns the result +:func json_loads: given string or bytes, loads and return its json using standard lib +:func trim_to_limit: returns a trimmed list if items if length exceeded limit given +:func clean_filename: returns the given string after removing no allowed characters +:func print_json: print json in formatted way, used for debug + +""" +import json +import math +import os +import re +import sys +import time + +import requests + +from . import settings +from .exceptions import ReqException + +sls = os.linesep + +_std_enabled = settings.LOG_STD +_inform_enabled = settings.LOG_INFORM +_save_enabled = settings.LOG_SAVE +_warn_enabled = settings.LOG_WARN + +__all__ = ['log', 'req', 'json_loads', 'trim_to_limit', 'clean_filename', 'print_json'] + + +def log(*objects, sep=' ', end='\n', file=sys.stdout, flush=True, start='', inform=False, save=False, error=False, + warn=False, normal=False): + """Print according to params and settings.py + + **Description** + settings.py's LOG_TYPE controls the overall behaviour of this function + eg. whether each type of log should be available + caller code controls the type of log + eg. whether the strings send to log should be type of inform + This function copied all params of python's print function, except flush is set to True, + and some custom parameters as shown below + + **Parameters** + :param start: + the string to print at the start, preceding all other string, including inform & save 's prefix + :type start: + string + + :param inform: + if this is True, a prefix ' >>>' is added at the front of the strings given, default False + :type inform: + boolean + + :param error: + if this is True, a prefix ' !!!' is added at the front of the strings given, default False + :type error: + boolean + + :param save: + if this is True, the strings given is also saved to LOG_FILE as specified in settings.py, default False + :type save: + boolean + + + """ + + if normal: + print(start, *objects, sep=sep, end=end, file=file, flush=flush) + return + + global _std_enabled, _inform_enabled, _save_enabled, _warn_enabled + if _inform_enabled and inform: + print(start, '>>>', *objects, sep=sep, end=end, file=file, flush=flush) + if _save_enabled and save: + print(start, *objects, sep=sep, end=end, file=open(settings.LOG_FILE, 'a', encoding='utf-8'), flush=False) + if _inform_enabled and error: + print(start, '!!!', *objects, sep=sep, end=end, file=file, flush=flush) + if _warn_enabled and warn: + print(start, '###', *objects, sep=sep, end=end, file=file, flush=flush) + if _std_enabled and not (inform or save or error or warn): + print(start, *objects, sep=sep, end=end, file=file, flush=flush) + + +# send request using requests, raise ReqException if fails all retries +def req(url, req_type='get', session=None, params=None, data=None, headers=settings.DEFAULT_HEADERS, + timeout=settings.TIMEOUT, err_msg=None, log_req=settings.LOG_REQUEST, retries=settings.MAX_RETRIES_FOR_REQUEST, + proxies=settings.REQUEST_PROXIES): + """Send requests according to given parameters using requests library + + **Description** + This function send request using requests library, + however its parameters does not accepts all parameters as in requests.get/post + and some custom parameters is added as shown below + + **Parameters** + :param url: + the url used for requesting + :type url: + string + + :param req_type: + the type of requests to send, given string is converted to uppercase before checking, default get + :type req_type: + string + + :param session: + if this is given, session.get/post is used instead of requests.get/post, default None + :type session: + requests.Session + + :param params: + the parameters send along request, default None + :type params: + same as params in requests library + + :param data: + the data send along when post method is used, default None + :type data: + same as data in requests library + + :param headers: + the headers send along when requesting, default None + :type headers: + same as headers in requests library + + :param timeout: + time out used when send requests, in seconds, default use settings.TIMEOUT + :type timeout: + int + + :param err_msg: + the error message used when requests.exceptions.RequestException is raised during requesting + :type err_msg: + string + + :param log_req: + specify whether to log the details of this request, default True + :type log_req: + boolean + + :param retries: + number of retries if request fails, if not given, settings.MAX_RETRIES_FOR_REQUEST is used + :type retries: + int + + :param proxies: + Proxies used for sending request, uses REQUEST_PROXIES in settings.py + :type proxies: + dict + + + **Returns** + :return: respond of the request + :rtype: requests.Response Object + + + **Raises** + :raises ReqException: if all retries fails or invalid type is given + + """ + req_type = req_type.upper() + curr_retries = 0 + while curr_retries < retries: + if log_req: + log(req_type + ':', str(url), 'with params:', str(params), end='') + try: + + # try send request according to parameters + if session: + if req_type == 'GET': + res = session.get(url=url, headers=headers, params=params, timeout=timeout, proxies=proxies) + elif req_type == 'POST': + res = session.post(url=url, headers=headers, params=params, timeout=timeout, data=data, + proxies=proxies) + else: + raise ReqException('Request type error:', req_type) + else: + if req_type == 'GET': + res = requests.get(url=url, headers=headers, params=params, timeout=timeout, proxies=proxies) + elif req_type == 'POST': + res = requests.post(url=url, headers=headers, params=params, timeout=timeout, data=data, + proxies=proxies) + else: + raise ReqException('Request type error:', req_type) + + if log_req: + log(res.status_code) + + # check if request result is normal + if res: + if res.status_code < 400: + if settings.DELAY_PER_REQUEST: + time.sleep(int(settings.DELAY_PER_REQUEST)) + return res + else: + log('Status code error:', res.status_code, 'retries:', curr_retries, save=True) + else: + log('Requests returned Falsey, retries:', curr_retries, save=True) + except requests.exceptions.Timeout as e: + log(req_type, url, params, 'Time Out:', curr_retries, save=True) + log('Reason:', str(e), save=True, inform=True) + except requests.exceptions.RequestException as e: + if err_msg: + log('RequestException:', err_msg, save=True) + else: + log(settings.DEFAULT_REQUEST_ERROR_MSG.format(type=req_type), save=True) + log('Reason:', str(e), 'Retries:', curr_retries, save=True) + + curr_retries += 1 + time.sleep(0.5) # dont retry again too fast + + # if still fails after all retries + exception_msg = str(req_type) + ' failed: ' + str(url) + ' params: ' + str(params) + raise ReqException(exception_msg) + + +# attempt to decode given json, raise JSONDecodeError if fails +def json_loads(text, encoding='utf-8'): + return json.loads(text, encoding=encoding) + + +# trim the given items length to given limit +def trim_to_limit(items, limit): + if items: + if limit: + num_of_items = len(items) + + if num_of_items == limit: + return items + + if num_of_items > limit: + items = items[:limit] + log('Trimmed', num_of_items, 'items =>', limit, 'items') + else: + log('Number of items are less than limit:', num_of_items, '<', limit, inform=True, save=True) + return items + + +# remove invalid file name characters for windows +def clean_filename(string): + return re.sub(r'[:<>"\\/|?*]', '', str(string)) + + +# used for testing +def print_json(json_obj): + print(json.dumps(json_obj, indent=4, ensure_ascii=False)) + + +def new_session(): + return requests.Session() + + +class Printer(object): + + def __init__(self): + self.is_first_print = True + self.last_percent = None + self.last_percent_time_left = None + self.last_percent_print_time = None + self.est_time_lefts = [0, 0, 0, 0, 0] + self.start_time = None + self.last_printed_line = None + + def print_progress(self, curr, total, msg=None): + curr_percent = math.floor(curr / total * 100) + curr_time = time.time() + if self.is_first_print: + est_time_left = float("inf") + self.is_first_print = False + self.last_percent_time_left = est_time_left + self.last_percent_print_time = curr_time + self.start_time = time.time() + elif self.last_percent == curr_percent: + est_time_left = self.last_percent_time_left + else: + bad_est_time_left = (curr_time - self.last_percent_print_time) / (curr_percent - self.last_percent) * ( + 100 - curr_percent) + self.est_time_lefts.append(bad_est_time_left) + self.est_time_lefts = self.est_time_lefts[1:] + percent_left = 100 - curr_percent + percent_diff = curr_percent - self.last_percent + chunk_left = round(percent_left / percent_diff) + if chunk_left < len(self.est_time_lefts): + est_time_left = sum(self.est_time_lefts[-chunk_left:]) / chunk_left if chunk_left != 0 else 0.00 + else: + est_time_left = sum(self.est_time_lefts) / len(self.est_time_lefts) + self.last_percent_time_left = est_time_left + self.last_percent_print_time = curr_time + + self.last_percent = curr_percent + + if est_time_left != 0.0: + progress_text = '{0} / {1} => {2}% | Time Left est. {3:.2f}s'.format(curr, total, curr_percent, + est_time_left) + else: + progress_text = '{0} / {1} => {2}% '.format(curr, total, curr_percent) + + if msg: + progress_text = progress_text + ' | ' + str(msg) + + if self.last_printed_line: + spaces = len(self.last_printed_line) + else: + spaces = 1 + + log(progress_text, end='', start=settings.CLEAR_LINE, inform=True) + self.last_printed_line = progress_text + + def print_done(self, msg=None): + if msg: + log(f' [ done ] => {msg}', normal=True) + else: # a float, time taken + if self.is_first_print: + log(' [ done ]', normal=True) + else: + log(' [ done ] => {0:.2f}s'.format(time.time() - self.start_time), normal=True) + self.is_first_print = True + self.start_time = None + self.last_percent = None + self.last_percent_print_time = None + self.last_percent_time_left = None + self.last_printed_line = None + self.est_time_lefts = [0, 0, 0] + + +printer = Printer() + + +def print_progress(curr, total, msg=None): + global printer + printer.print_progress(curr, total, msg) + + +def print_done(msg=None): + global printer + printer.print_done(msg) diff --git a/gui/login.py b/gui/login.py new file mode 100644 index 0000000..6920b2f --- /dev/null +++ b/gui/login.py @@ -0,0 +1,55 @@ +from factory import make_label, make_entry, make_button, NORMAL, pack, CENTER, BOTTOM, make_text, DISABLED +from lib.pikax.exceptions import PikaxException + +from models import PikaxGuiComponent +from common import go_to_next_screen, StdoutRedirector +import sys + + +class LoginScreen(PikaxGuiComponent): + + def __init__(self, master, pikax_handler): + super().__init__(master, pikax_handler) + self.title_label = make_label(self.frame, text='Pikax') + self.title_label.configure(font="-weight bold", anchor=CENTER) + self.username_label = make_label(self.frame, text='username') + self.password_label = make_label(self.frame, text='password') + self.username_entry = make_entry(self.frame) + self.password_entry = make_entry(self.frame) + self.login_button = make_button(self.frame, text='login') + self.login_button.configure(command=self.login) + self.username_entry.bind('', self.login) + self.password_entry.bind('', self.login) + self.output_text = make_entry(self.frame) + self.output_text.configure(state=DISABLED, justify=CENTER) + sys.stdout = StdoutRedirector(self.output_text) + self.load() + + def login(self, event=None): + username = self.username_entry.get() + password = self.password_entry.get() + try: + self.pikax_handler.login(username, password) + from menu import MenuScreen + go_to_next_screen(src=self, dest=MenuScreen) + except PikaxException: + print('Login Failed') + + def load(self): + self.frame.pack_configure(expand=True) + self.output_text.pack_configure(side=BOTTOM, expand=True) + + pack(self.title_label) + pack(self.username_label) + pack(self.username_entry) + pack(self.password_label) + pack(self.password_entry) + pack(self.login_button) + pack(self.output_text) + pack(self.frame) + + self.username_entry.focus() + self.login_button.configure(state=NORMAL) + + def destroy(self): + self.frame.destroy() diff --git a/gui/main.py b/gui/main.py new file mode 100644 index 0000000..edc35f4 --- /dev/null +++ b/gui/main.py @@ -0,0 +1,22 @@ +from tkinter import * + +from login import LoginScreen +from pikaxhandler import PikaxHandler + +root = Tk() +root.geometry('400x400') +root.title('Pikax - Pixiv Downloader') + +pikax_handler = PikaxHandler() + + +# restorecyclebin@gmail.com + + +def main(): + login_screen = LoginScreen(master=root, pikax_handler=pikax_handler) + root.mainloop() + + +if __name__ == '__main__': + main() diff --git a/gui/menu.py b/gui/menu.py new file mode 100644 index 0000000..4240e31 --- /dev/null +++ b/gui/menu.py @@ -0,0 +1,56 @@ +from factory import make_button, NORMAL, pack +from models import PikaxGuiComponent +from common import go_to_next_screen + + +class MenuScreen(PikaxGuiComponent): + def __init__(self, master, pikax_handler): + super().__init__(master, pikax_handler) + self.search_button = make_button(self.frame, text='search') + self.rank_button = make_button(self.frame, text='rank') + self.rank_button.configure(command=self.rank_clicked) + self.search_button = make_button(self.frame, text='search') + self.search_button.configure(command=self.search_clicked) + self.back_button = make_button(self.frame, text='back') + self.back_button.configure(command=self.back_clicked) + self.load() + + def rank_clicked(self): + from rank import RankScreen + go_to_next_screen(src=self, dest=RankScreen) + + def search_clicked(self): + from search import SearchScreen + go_to_next_screen(src=self, dest=SearchScreen) + + def back_clicked(self): + from login import LoginScreen + go_to_next_screen(src=self, dest=LoginScreen) + + def load(self): + self.frame.pack_configure(expand=True) + + pack(self.search_button) + pack(self.rank_button) + pack(self.back_button) + pack(self.frame) + + self.search_button.configure(state=NORMAL) + self.rank_button.configure(state=NORMAL) + self.search_button.configure(state=NORMAL) + self.back_button.configure(state=NORMAL) + + def destroy(self): + self.frame.destroy() + + +def main(): + from pikaxhandler import PikaxHandler + import tkinter as tk + root = tk.Tk() + MenuScreen(root, pikax_handler=PikaxHandler()) + root.mainloop() + + +if __name__ == '__main__': + main() diff --git a/gui/models.py b/gui/models.py new file mode 100644 index 0000000..79c17d5 --- /dev/null +++ b/gui/models.py @@ -0,0 +1,22 @@ +from tkinter import Frame + + +class PikaxGuiComponent: + + def __init__(self, master, pikax_handler): + self._frame = Frame(master) + self._pikax_handler = pikax_handler + + def load(self): + raise NotImplementedError + + def destroy(self): + raise NotImplementedError + + @property + def frame(self): + return self._frame + + @property + def pikax_handler(self): + return self._pikax_handler diff --git a/gui/pikaxhandler.py b/gui/pikaxhandler.py new file mode 100644 index 0000000..4a590e4 --- /dev/null +++ b/gui/pikaxhandler.py @@ -0,0 +1,26 @@ +from lib.pikax.exceptions import PikaxException +from lib.pikax.items import LoginHandler +from lib.pikax.pikax import Pikax + + +class PikaxHandler: + def __init__(self): + self.pikax = Pikax() + self.user = None + + def login(self, username, password): + status, client = LoginHandler().android_login(username, password) + if status is LoginHandler.LoginStatus.ANDROID: + self.pikax.android_client = client + else: + raise PikaxException('Failed Login') + + def rank(self): + ... + + def search(self, keyword, limit, sort, match, popularity, folder): + + result = self.pikax.search(keyword=keyword, limit=limit, sort=sort, match=match, popularity=popularity) + for curr, total, info in self.pikax.download(result, folder): + print(curr, total, info) + diff --git a/gui/rank.py b/gui/rank.py new file mode 100644 index 0000000..249412a --- /dev/null +++ b/gui/rank.py @@ -0,0 +1,13 @@ +from models import PikaxGuiComponent + + +class RankScreen(PikaxGuiComponent): + + def __init__(self, master, pikax_handler): + super().__init__(master, pikax_handler) + + def load(self): + ... + + def destroy(self): + ... diff --git a/gui/search.py b/gui/search.py new file mode 100644 index 0000000..c106a6d --- /dev/null +++ b/gui/search.py @@ -0,0 +1,153 @@ +import os +import sys +from tkinter import ttk + +from common import go_to_next_screen, StdoutRedirector +from factory import make_label, make_entry, make_button, make_dropdown, NORMAL, grid, pack, DISABLED, make_text +from lib.pikax.util import clean_filename +from menu import MenuScreen +from models import PikaxGuiComponent +from lib import pikax + + +class SearchScreen(PikaxGuiComponent): + + def __init__(self, master, pikax_handler): + super().__init__(master, pikax_handler) + self.keyword_label = make_label(self.frame, text='keyword') + self.match_label = make_label(self.frame, text='tag match') + self.sort_label = make_label(self.frame, text='sort') + self.popularity_label = make_label(self.frame, text='popularity') + self.limit_label = make_label(self.frame, text='limit') + + self.keyword_entry = make_entry(self.frame) + self.limit_entry = make_entry(self.frame) + + self.sort_choices = ['date ascending', 'date descending'] + self.sort_dropdown = make_dropdown(self.frame, 'date descending', self.sort_choices) + + self.popularity_choices = ['any', '100', '500', '1000', '5000', '10000', '20000'] + self.popularity_dropdown = make_dropdown(self.frame, 'any', self.popularity_choices) + + self.match_choices = ['exact', 'partial', 'any'] + self.match_dropdown = make_dropdown(self.frame, 'partial', self.match_choices) + + self.folder_label = make_label(self.frame, text='download folder') + self.folder_entry = make_entry(self.frame) + + self.search_and_download_button = make_button(self.frame, text='search and download') + self.search_and_download_button.configure(command=self.search_and_download_clicked) + self.back_button = make_button(self.frame, text='back') + self.back_button.configure(command=self.back_clicked) + + self.output_text = make_text(self.frame) + self.output_text.configure(state=DISABLED) + sys.stdout = StdoutRedirector(self.output_text) + + self.load() + + def load(self): + self.keyword_label.grid_configure(row=0) + self.limit_label.grid_configure(row=1) + self.sort_label.grid_configure(row=2) + self.popularity_label.grid_configure(row=3) + self.match_label.grid_configure(row=4) + self.folder_label.grid_configure(row=5) + + self.keyword_entry.grid_configure(row=0, column=1) + self.limit_entry.grid_configure(row=1, column=1) + self.sort_dropdown.grid_configure(row=2, column=1) + self.popularity_dropdown.grid_configure(row=3, column=1) + self.match_dropdown.grid_configure(row=4, column=1) + self.folder_entry.grid_configure(row=5, column=1) + + self.back_button.grid_configure(row=6, column=0) + self.back_button.configure(state=NORMAL) + + self.search_and_download_button.grid_configure(row=6, column=1) + self.search_and_download_button.configure(state=NORMAL) + + self.output_text.grid_configure(row=7, columnspan=2) + self.output_text.configure(height=6) + + self.frame.pack_configure(expand=True) + + grid(self.keyword_label) + grid(self.keyword_entry) + grid(self.match_label) + grid(self.match_dropdown) + grid(self.limit_label) + grid(self.limit_entry) + grid(self.folder_label) + grid(self.folder_entry) + grid(self.popularity_label) + grid(self.popularity_dropdown) + grid(self.sort_label) + grid(self.sort_dropdown) + grid(self.output_text) + + pack(self.frame) + + def destroy(self): + self.frame.destroy() + + def back_clicked(self): + go_to_next_screen(src=self, dest=MenuScreen) + + def search_and_download_clicked(self): + try: + keyword = str(self.keyword_entry.get() or '') + limit_input = int(self.limit_entry.get()) if self.limit_entry.get() else None + match_input = str(self.match_dropdown.get()) + sort_input = str(self.sort_dropdown.get()) + popularity_input = str(self.popularity_dropdown.get()) + folder_input = str(self.folder_entry.get()) + params = self.check_inputs(limit_input=limit_input, match_input=match_input, sort_input=sort_input, + popularity_input=popularity_input, folder_input=folder_input) + except (TypeError, ValueError) as e: + print('Please check your inputs', os.linesep, f'Error Message: {e}') + return + + import threading + params['keyword'] = keyword + download_thread = threading.Thread(target=self.pikax_handler.search, kwargs=params) + download_thread.start() + + @staticmethod + def check_inputs(limit_input, match_input, sort_input, popularity_input, folder_input): + from lib.pikax import params + + if not limit_input or limit_input == 'any': + limit = None + else: + limit = int(limit_input) + + if not match_input or match_input == 'any': + match = params.Match.ANY + elif match_input == 'exact': + match = params.Match.EXACT + else: + match = params.Match.PARTIAL + + if not sort_input or sort_input == 'date descending': + sort = params.Sort.DATE_DESC + else: + sort = params.Sort.DATE_ASC + + if not popularity_input or popularity_input == 'any': + popularity = None + else: + popularity = int(popularity_input) + + if folder_input: + folder = clean_filename(folder_input) + else: + folder = None + + return { + 'limit': limit, + 'sort': sort, + 'match': match, + 'popularity': popularity, + 'folder': folder + }