diff --git a/README.md b/README.md index 0293e4f..217ea5b 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ [![GitHub downloads](https://img.shields.io/github/downloads/KurtBestor/Hitomi-Downloader/latest/total.svg?logo=github)](https://github.com/KurtBestor/Hitomi-Downloader/releases/latest) [![GitHub downloads](https://img.shields.io/github/downloads/KurtBestor/Hitomi-Downloader/total.svg?logo=github)](https://github.com/KurtBestor/Hitomi-Downloader/releases) +## Links - [Download](https://github.com/KurtBestor/Hitomi-Downloader/releases/latest) - [Issues](https://github.com/KurtBestor/Hitomi-Downloader/issues) - [Scripts](https://github.com/KurtBestor/Hitomi-Downloader/wiki/Scripts) @@ -15,6 +16,19 @@ ## Demo +## Features +- 🍰 Simple and clear user interface +- 🚀 Download acceleration +- 💻 Supports 32 threads in a single task +- 🚥 Supports speed limit +- 📜 Supports user scripts +- 🧲 Supports BitTorrent & Magnet +- 🎞️ Supports M3U8 & MPD format videos +- 🌙 Dark mode +- 🧳 Portable +- 📋 Clipboard monitor +- 🗃️ Easy to organize tasks + ## Supported Sites | Site | URL | | :--: | -- | @@ -32,6 +46,7 @@ | **Epio** | | | **E(x)Hentai Galleries** |
| | **Facebook** | | +| **FC2 Video** | | | **Flickr** | | | **Gelbooru** | | | **hanime.tv** | | @@ -44,6 +59,7 @@ | **Jmana** | | | **カクヨム** | | | **LHScan** | | +| **Likee** | | | **Luscious** | | | **Manamoa** | | | **MyReadingManga** | | @@ -51,6 +67,7 @@ | **Naver Post** | | | **Naver Webtoon** | | | **nhentai** | | +| **nhentai.com** | | | **Niconico** | | | **ニジエ** | | | **Pawoo** | | @@ -75,3 +92,4 @@ | **XVideos** | | | **Yande.re** | | | **YouTube** | | +| **and more...** | [Supported sites by youtube-dl](http://ytdl-org.github.io/youtube-dl/supportedsites.html) | diff --git a/src/extractor/afreeca_downloader.py b/src/extractor/afreeca_downloader.py index 5e94f63..e970051 100644 --- a/src/extractor/afreeca_downloader.py +++ b/src/extractor/afreeca_downloader.py @@ -31,9 +31,6 @@ class Downloader_afreeca(Downloader): single = True display_name = 'AfreecaTV' - def init(self): - self.url = self.url.replace('afreeca_', '') - def read(self): session = Session() video = get_video(self.url, session) diff --git a/src/extractor/artstation_downloader.py b/src/extractor/artstation_downloader.py index b674ea9..cba10e6 100644 --- a/src/extractor/artstation_downloader.py +++ b/src/extractor/artstation_downloader.py @@ -28,7 +28,7 @@ class Downloader_artstation(Downloader): display_name = 'ArtStation' def init(self): - self.url_main = 'https://www.artstation.com/{}'.format(self.id.replace('artstation_', '').replace('／', '/')) + self.url_main = 'https://www.artstation.com/{}'.format(self.id.replace('artstation_', '', 1).replace('／', '/')) if '/artwork/' in self.url: pass#raise NotImplementedError('Single post') @@ -53,7 +53,7 @@ def name(self): def read(self): cw = self.customWidget self.title = self.name - id = self.id.replace('artstation_', '').replace('／', '/') + id = self.id.replace('artstation_', '', 1).replace('／', '/') if '/' in id: type = id.split('/')[1] id = id.split('/')[0] diff --git a/src/extractor/asiansister_downloader.py b/src/extractor/asiansister_downloader.py index 10ab14f..fb28ded 100644 --- a/src/extractor/asiansister_downloader.py +++ b/src/extractor/asiansister_downloader.py @@ -13,7 +13,6 @@ class Downloader_asiansister(Downloader): @try_n(4) def init(self): - self.url = self.url.replace('asiansister_', '') html = downloader.read_html(self.url) self.soup = Soup(html) diff --git a/src/extractor/asmhentai_downloader.py b/src/extractor/asmhentai_downloader.py index b8dd93b..37eec0d 100644 --- a/src/extractor/asmhentai_downloader.py +++ b/src/extractor/asmhentai_downloader.py @@ -28,7 +28,6 @@ def init(self): @classmethod def fix_url(cls, url): - url = url.replace('asmhentai_', '') id_ = get_id(url) return 'https://asmhentai.com/g/{}/'.format(id_) diff --git a/src/extractor/avgle_downloader.py b/src/extractor/avgle_downloader.py index 1e6d487..de4e099 100644 --- a/src/extractor/avgle_downloader.py +++ b/src/extractor/avgle_downloader.py @@ -18,7 +18,6 @@ class Downloader_avgle(Downloader): URLS = ['avgle.com'] def init(self): - self.url = self.url.replace('avgle_', '', 1) if not self.customWidget.data_: link = 'https://github.com/KurtBestor/Hitomi-Downloader/wiki/Chrome-Extension' webbrowser.open(link) diff --git a/src/extractor/baraag_downloader.py b/src/extractor/baraag_downloader.py index 52e1a51..fd4a872 100644 --- a/src/extractor/baraag_downloader.py +++ b/src/extractor/baraag_downloader.py @@ -21,7 +21,6 @@ def init(self): @classmethod def fix_url(cls, url): - url = url.replace('baraag_', '') id_ = get_id(url) or url return 'https://baraag.net/{}'.format(id_) diff --git a/src/extractor/bcy_downloader.py b/src/extractor/bcy_downloader.py index bd11cf5..ae63774 100644 --- a/src/extractor/bcy_downloader.py +++ b/src/extractor/bcy_downloader.py @@ -16,7 +16,6 @@ class Downloader_bcy(Downloader): display_name = '半次元' def init(self): - self.url = self.url.replace('bcy_', '') self.html = downloader.read_html(self.url) self.info = get_info(self.url, self.html) diff --git a/src/extractor/bdsmlr_downloader.py b/src/extractor/bdsmlr_downloader.py index fabfe6e..6f20e45 100644 --- a/src/extractor/bdsmlr_downloader.py +++ b/src/extractor/bdsmlr_downloader.py @@ -17,8 +17,6 @@ class Downloader_bdsmlr(Downloader): display_name = 'BDSMlr' def init(self): - self.url = self.url.replace('bdsmlr_', '') - if u'bdsmlr.com/post/' in self.url: return self.Invalid(tr_(u'개별 다운로드는 지원하지 않습니다: {}').format(self.url), fail=False) diff --git a/src/extractor/bili_downloader.py b/src/extractor/bili_downloader.py index 59b7bb5..78769fa 100644 --- a/src/extractor/bili_downloader.py +++ b/src/extractor/bili_downloader.py @@ -69,8 +69,8 @@ class Downloader_bili(Downloader): def init(self): self.url = fix_url(self.url, self.customWidget) - if 'bili_' in self.url: - self.url = u'https://www.bilibili.com/video/{}'.format(self.url.replace('bili_', '')) + if 'bilibili.com' not in self.url.lower(): + self.url = 'https://www.bilibili.com/video/{}'.format(self.url) self.url = self.url.replace('m.bilibili', 'bilibili') @property @@ -96,7 +96,7 @@ def read(self): title += (u'_p{}').format(page) title = format_filename(title, self.id_, '.mp4')[:-4] n = int(math.ceil(8.0 / len(videos))) - self.customWidget.print_(('n_threads: {}').format(n)) + self.print_(('n_threads: {}').format(n)) self.enableSegment(n_threads=n) self.title = title diff --git a/src/extractor/comicwalker_downloader.py b/src/extractor/comicwalker_downloader.py index caee8b7..f7d3cd9 100644 --- a/src/extractor/comicwalker_downloader.py +++ b/src/extractor/comicwalker_downloader.py @@ -50,9 +50,6 @@ class Downloader_comicwalker(Downloader): display_name = 'ComicWalker' _soup = None pages = None - - def init(self): - self.url = url = self.url.replace('comicwalker_', '') @property def soup(self): diff --git a/src/extractor/danbooru_downloader.py b/src/extractor/danbooru_downloader.py index e95fa88..171f526 100644 --- a/src/extractor/danbooru_downloader.py +++ b/src/extractor/danbooru_downloader.py @@ -20,7 +20,6 @@ class Downloader_danbooru(Downloader): _name = None def init(self): - self.url = self.url.replace('danbooru_', '') if 'donmai.us' in self.url: self.url = self.url.replace('http://', 'https://') else: diff --git a/src/extractor/discord_emoji_downloader.py b/src/extractor/discord_emoji_downloader.py index 21f4e2f..f9b4f73 100644 --- a/src/extractor/discord_emoji_downloader.py +++ b/src/extractor/discord_emoji_downloader.py @@ -37,7 +37,7 @@ def init(self): pass def read(self): - token_guild_id_list = self.url.replace("discord_", "", 1).split( + token_guild_id_list = self.url.split( "/" ) # 값을 어떻게 받을지 몰라서 일단 나눴어요. discord_이메일/비밀번호/서버아이디 또는 discord_토큰/서버아이디 이런식으로 받게 해놨어요. diff --git a/src/extractor/gelbooru_downloader.py b/src/extractor/gelbooru_downloader.py index a79ca2e..592400c 100644 --- a/src/extractor/gelbooru_downloader.py +++ b/src/extractor/gelbooru_downloader.py @@ -36,7 +36,6 @@ class Downloader_gelbooru(Downloader): _name = None def init(self): - self.url = self.url.replace('gelbooru_', '') if 'gelbooru.com' in self.url.lower(): self.url = self.url.replace('http://', 'https://') else: diff --git a/src/extractor/hameln_downloader.py b/src/extractor/hameln_downloader.py index 0fe6555..ef8f279 100644 --- a/src/extractor/hameln_downloader.py +++ b/src/extractor/hameln_downloader.py @@ -18,7 +18,6 @@ class Downloader_hameln(Downloader): detect_removed = False def init(self): - self.url = self.url.replace('hameln_', '') id_ = re.find('/novel/([^/]+)', self.url) if id_ is not None: self.url = 'https://syosetu.org/novel/{}/'.format(id_) diff --git a/src/extractor/hanime_downloader.py b/src/extractor/hanime_downloader.py index 737cb35..345e463 100644 --- a/src/extractor/hanime_downloader.py +++ b/src/extractor/hanime_downloader.py @@ -41,10 +41,6 @@ class Downloader_hanime(Downloader): single = True display_name = 'hanime.tv' - def init(self): - if self.url.startswith('hanime_'): - self.url = self.url.replace('hanime_', '', 1) - def read(self): cw = self.customWidget video, session = get_video(self.url) diff --git a/src/extractor/hf_downloader.py b/src/extractor/hf_downloader.py index 47fda9c..fed5b85 100644 --- a/src/extractor/hf_downloader.py +++ b/src/extractor/hf_downloader.py @@ -44,8 +44,6 @@ def f(_): def get_username(url): if 'user/' in url: username = url.split('user/')[1].split('?')[0].split('/')[0] - else: - username = url.replace('hf_', '') return username diff --git a/src/extractor/manatoki_downloader.py b/src/extractor/manatoki_downloader.py index 842810f..b5156fc 100644 --- a/src/extractor/manatoki_downloader.py +++ b/src/extractor/manatoki_downloader.py @@ -34,8 +34,6 @@ class Downloader_manatoki(Downloader): @try_n(2) def init(self): - self.url = self.url.replace('manatoki_', '') - self.session, self.soup, url = get_soup(self.url) self.url = self.fix_url(url) diff --git a/src/extractor/naverpost_downloader.py b/src/extractor/naverpost_downloader.py index d25d289..79f12b4 100644 --- a/src/extractor/naverpost_downloader.py +++ b/src/extractor/naverpost_downloader.py @@ -53,7 +53,6 @@ class DownloaderNaverPost(Downloader): URLS = ["m.post.naver.com", "post.naver.com"] def init(self): - self.url = self.url.replace("naver_post_", "") self.parsed_url = urlparse(self.url) # url 나눔 self.soup = get_soup(self.url) diff --git a/src/extractor/nhentai_downloader.py b/src/extractor/nhentai_downloader.py index 7b8c704..d292492 100644 --- a/src/extractor/nhentai_downloader.py +++ b/src/extractor/nhentai_downloader.py @@ -15,7 +15,6 @@ class Downloader_nhentai(Downloader): display_name = 'nhentai' def init(self): - self.url = self.url.replace('nhentai_', '') self.url = 'https://nhentai.net/g/{}/'.format(self.id_) @property diff --git a/src/extractor/pawoo_downloader.py b/src/extractor/pawoo_downloader.py index cd2969c..ee438ce 100644 --- a/src/extractor/pawoo_downloader.py +++ b/src/extractor/pawoo_downloader.py @@ -13,7 +13,6 @@ class Downloader_pawoo(Downloader): URLS = ['pawoo.net'] def init(self): - self.url = self.url.replace('pawoo_', '') self.url = 'https://pawoo.net/{}'.format(self.id_) self.referer = self.url diff --git a/src/extractor/pixiv_comic_downloader.py b/src/extractor/pixiv_comic_downloader.py index 20e74af..bf88a54 100644 --- a/src/extractor/pixiv_comic_downloader.py +++ b/src/extractor/pixiv_comic_downloader.py @@ -33,7 +33,6 @@ class Downloader_pixiv_comic(Downloader): display_name = 'pixivコミック' def init(self): - self.url = self.url.replace('pixiv_comic_', '') if '/viewer/' in self.url: html = downloader.read_html(self.url) id = re.find('/works/([0-9]+)', html) diff --git a/src/extractor/pixiv_downloader.py b/src/extractor/pixiv_downloader.py index 7ad078f..da23863 100644 --- a/src/extractor/pixiv_downloader.py +++ b/src/extractor/pixiv_downloader.py @@ -44,6 +44,7 @@ class Downloader_pixiv(Downloader): info = None _id = None keep_date = True + strip_header = False atts = ['_format', '_format_name', 'imgs'] def init(self): @@ -51,13 +52,13 @@ def init(self): url = self.url # Determine the type - if 'bookmark.php?type=user' in url or headers['following'] in url: + if 'bookmark.php?type=user' in url or url.startswith(headers['following']): type = 'following' - elif 'bookmark.php' in url or headers['bookmark'] in url or '/bookmarks/' in url: + elif 'bookmark.php' in url or url.startswith(headers['bookmark']) or '/bookmarks/' in url: type = 'bookmark' - elif 'illust_id=' in url or headers['illust'] in url or '/artworks/' in url: + elif 'illust_id=' in url or url.startswith(headers['illust']) or '/artworks/' in url: type = 'illust' - elif 'search.php' in url or headers['search'] in url: + elif 'search.php' in url or url.startswith(headers['search']): type = 'search' order = query_url(url).get('order', ['date_d'])[0] # data_d, date, popular_d, popular_male_d, popular_female_d scd = query_url(url).get('scd', [None])[0] # 2019-09-27 @@ -91,7 +92,7 @@ def init(self): 'blt': blt, 'bgt': bgt, 'type': type_} - elif 'id=' in url and 'mode=' not in url or headers['user'] in url or 'pixiv.me' in url or '/users/' in url: + elif 'id=' in url and 'mode=' not in url or url.startswith(headers['user']) or 'pixiv.me' in url or '/users/' in url: type = 'user' else: self.Invalid((u'[pixiv] Can not determine type: {}').format(url)) diff --git a/src/extractor/sankaku_downloader.py b/src/extractor/sankaku_downloader.py index dbb22fb..bdee973 100644 --- a/src/extractor/sankaku_downloader.py +++ b/src/extractor/sankaku_downloader.py @@ -28,7 +28,6 @@ class Downloader_sankaku(Downloader): display_name = 'Sankaku Complex' def init(self): - self.url = self.url.replace('sankaku_', '') if '/post/' in self.url: return self.Invalid('Single post is not supported') diff --git a/src/extractor/syosetu_downloader.py b/src/extractor/syosetu_downloader.py index f96e6c3..4ecdd72 100644 --- a/src/extractor/syosetu_downloader.py +++ b/src/extractor/syosetu_downloader.py @@ -40,7 +40,6 @@ class Downloader_syosetu(Downloader): display_name = '小説家になろう' def init(self): - self.url = self.url.replace('syosetu_', '') self.url = (u'https://ncode.syosetu.com/{}/').format(self.id_) @property diff --git a/src/extractor/torrent_downloader.py b/src/extractor/torrent_downloader.py index 04539a9..a3e4c8a 100644 --- a/src/extractor/torrent_downloader.py +++ b/src/extractor/torrent_downloader.py @@ -28,8 +28,6 @@ def init(self): global torrent if torrent is None: import torrent - if self.url.startswith('torrent_'): - self.url = self.url.replace('torrent_', '', 1) @property def name(self): diff --git a/src/extractor/twitch_downloader.py b/src/extractor/twitch_downloader.py index 004e0b6..3622b18 100644 --- a/src/extractor/twitch_downloader.py +++ b/src/extractor/twitch_downloader.py @@ -21,7 +21,7 @@ def init(self): url = 'https://' + url self.url = url else: - url = 'https://www.twitch.tv/videos/{}'.format(url.replace('twitch_', '')) + url = 'https://www.twitch.tv/videos/{}'.format(url) self.url = url @classmethod diff --git a/src/extractor/twitter_downloader.py b/src/extractor/twitter_downloader.py index 904791c..ce74cf3 100644 --- a/src/extractor/twitter_downloader.py +++ b/src/extractor/twitter_downloader.py @@ -67,7 +67,6 @@ class Downloader_twitter(Downloader): def init(self): self.session = get_session() - self.url = self.url.replace('twitter_', '') #self.url = fix_url(self.url) self.artist, self.username = get_artist_username(self.url, self.session) if self.username == 'home': @@ -84,7 +83,7 @@ def fix_url(cls, url): @classmethod def key_id(cls, url): - return cls.fix_url(url).lower() + return url.lower() def read(self): cw = self.customWidget @@ -126,7 +125,10 @@ class TwitterAPI(object): def __init__(self, session, cw=None): self.session = session self.cw = cw - csrf = hashlib.md5(str(time()).encode()).hexdigest() + csrf = session.cookies.get('ct0', domain='.twitter.com') + print('csrf:', csrf) + if not csrf: + csrf = hashlib.md5(str(time()).encode()).hexdigest() hdr = { "authorization": AUTH, "x-twitter-client-language": "en", @@ -182,7 +184,11 @@ def _call(self, url_api, referer='https://twitter.com', params=None): if params: url_api = update_url_query(url_api, params) #print('call:', url_api) - data = downloader.read_json(url_api, referer, session=self.session) + r = self.session.get(url_api, headers={'Referer': referer}) + csrf = r.cookies.get('ct0') + if csrf: + self.session.headers['x-csrf-token'] = csrf + data = json.loads(r.text) return data def search(self, query): @@ -231,6 +237,9 @@ def _pagination(self, url_api, params=None, entry_tweet="tweet-", entry_cursor=" for try_ in range(n_try): try: data = self._call(url_api, params=params) + if 'globalObjects' not in data: + try_ = n_try + raise Exception(str(data['errors'])) tweets = data["globalObjects"]["tweets"] break except Exception as e: diff --git a/src/extractor/wikiart_downloader.py b/src/extractor/wikiart_downloader.py index c9dde4f..2351788 100644 --- a/src/extractor/wikiart_downloader.py +++ b/src/extractor/wikiart_downloader.py @@ -25,8 +25,6 @@ class Downloader_wikiart(Downloader): display_name = 'WikiArt' def init(self): - self.url = self.url.replace('wikiart_', '') - self.url = u'https://www.wikiart.org/en/{}'.format(self.id_) html = downloader.read_html(self.url) self.soup = Soup(html) diff --git a/src/extractor/worldcos_downloader.py b/src/extractor/worldcos_downloader.py index 9e380f5..7578059 100644 --- a/src/extractor/worldcos_downloader.py +++ b/src/extractor/worldcos_downloader.py @@ -23,7 +23,6 @@ class Downloader_worldcos(Downloader): display_name = 'World Cosplay' def init(self): - self.url = self.url.replace('worldcos_', '') if 'worldcosplay.net' in self.url.lower(): self.url = self.url.replace('http://', 'https://') else: diff --git a/src/extractor/xhamster_downloader.py b/src/extractor/xhamster_downloader.py index 11db5b0..0679b8c 100644 --- a/src/extractor/xhamster_downloader.py +++ b/src/extractor/xhamster_downloader.py @@ -1,5 +1,5 @@ import downloader, ree as re -from utils import Downloader, get_outdir, Soup, get_p2f, LazyUrl, get_print, cut_pair, get_ext, try_n, format_filename, clean_title +from utils import Downloader, get_outdir, Soup, LazyUrl, get_print, cut_pair, get_ext, try_n, format_filename, clean_title from timee import sleep from error_printer import print_error import os @@ -20,7 +20,6 @@ class Downloader_xhamster(Downloader): display_name = 'xHamster' def init(self): - self.url = self.url.replace('xhamster_', '') if re.search(r'xhamsterlive[0-9]*\.', self.url): raise Exception('xHamsterLive') if not re.search(r'xhamster[0-9]*\.', self.url): @@ -40,22 +39,8 @@ def read(self): if '/users/' in self.url: info = read_channel(self.url, cw) urls = info['urls'] - p2f = get_p2f(cw) - if p2f: - self.single = False - self.title = clean_title(info['title']) - videos = [Video(url) for url in urls] - self.urls = [video.url for video in videos] - video = videos[0] - video.url() - downloader.download(video.info['thumbnail'], buffer=thumb) - self.setIcon(thumb) - return - else: - cw.gal_num = self.url = urls.pop(0) - if urls and cw.alive: - s = u', '.join(urls) - self.exec_queue.put((s, 'downButton(customWidget)')) + videos = [Video(url) for url in urls] + video = self.process_playlist(info['title'], videos) elif '/photos/gallery/' in self.url: info = read_gallery(self.url, cw) for img in info['imgs']: @@ -66,21 +51,20 @@ def read(self): self.disableSegment() return else: - urls = [] - video = Video(self.url) - video.url() - self.urls.append(video.url) + video = Video(self.url) + video.url() + self.urls.append(video.url) + self.title = video.title downloader.download(video.info['thumbnail'], buffer=thumb) self.setIcon(thumb) - self.title = video.title class Video(object): _url = None def __init__(self, url): - url = downloader.real_url(url) + #url = downloader.real_url(url) self.url = LazyUrl(url, self.get, self) @try_n(2) @@ -130,36 +114,56 @@ def get_info(url): return info +def read_page(username, p, cw): + print_ = get_print(cw) + url = 'https://xhamster.com/users/{}/videos/{}'.format(username, p) + print_(url) + n = 4 + for try_ in range(n): + try: + soup = downloader.read_soup(url) + items = soup.findAll('div', class_='thumb-list__item') + if not items and try_ < n-1: + continue + break + except Exception as e: + e_ = e + print(e) + else: + raise e_ + return items + + def read_channel(url, cw=None): print_ = get_print(cw) username = url.split('/users/')[1].split('/')[0] info = {} - html = downloader.read_html(url) - soup = Soup(html) + soup = downloader.read_soup(url) title = soup.find('div', class_='user-name').text.strip() info['title'] = u'[Channel] {}'.format(title) - items = [] - for p in range(1, 21): - url = 'https://xhamster.com/users/{}/videos/{}'.format(username, p) - print_(url) - html = downloader.read_html(url) - soup = Soup(html) - items_ = soup.findAll('div', class_='thumb-list__item') - if not items_: + urls = [] + urls_set = set() + for p in range(1, 101): + items = read_page(username, p, cw) + if not items: print('no items') break - for item in items_: - items.append(item) - - urls = [] - for item in items: - url = item.a.attrs['href'] - if url in urls: - print('duplicate:', url) - continue - urls.append(url) + for item in items: + if item.find('span', class_='thumb-image-container__status-text'): #2858 + continue + url = item.a.attrs['href'] + if url in urls_set: + print('duplicate:', url) + continue + urls_set.add(url) + urls.append(url) + s = '{} {} - {}'.format(tr_('읽는 중...'), info['title'], len(urls)) + if cw: + cw.setTitle(s) + else: + print(s) info['urls'] = urls @@ -194,8 +198,7 @@ def read_gallery(url, cw=None): info = {} - html = downloader.read_html(url) - soup = Soup(html) + soup = downloader.read_soup(url) h1 = soup.find('h1') if h1.find('a'): diff --git a/src/extractor/xnxx_downloader.py b/src/extractor/xnxx_downloader.py index e8b5265..f51f0c3 100644 --- a/src/extractor/xnxx_downloader.py +++ b/src/extractor/xnxx_downloader.py @@ -37,9 +37,6 @@ class Downloader_xnxx(Downloader): single = True display_name = 'XNXX' - def init(self): - self.url = self.url.replace('xnxx_', '') - def read(self): video = get_video(self.url) self.urls.append(video.url) diff --git a/src/extractor/xvideo_downloader.py b/src/extractor/xvideo_downloader.py index 769eaee..0266d6f 100644 --- a/src/extractor/xvideo_downloader.py +++ b/src/extractor/xvideo_downloader.py @@ -1,5 +1,5 @@ import downloader -from utils import Downloader, Soup, LazyUrl, urljoin, format_filename, clean_title, Session, get_ext, get_p2f, get_print, get_max_range +from utils import Downloader, Soup, LazyUrl, urljoin, format_filename, clean_title, Session, get_ext, get_print, get_max_range from io import BytesIO from constants import try_n import ree as re @@ -49,7 +49,6 @@ class Downloader_xvideo(Downloader): display_name = 'XVideos' def init(self): - self.url = self.url.replace('xvideo_', '') if 'xvideos.' in self.url.lower(): self.url = self.url.replace('http://', 'https://') else: @@ -65,31 +64,22 @@ def key_id(cls, url): res = re.find(CHANNEL_PATTERN, url) if res: return '_'.join(res) - return self.fix_url(url) + return url def read(self): cw = self.customWidget + res = re.find(CHANNEL_PATTERN, self.url) if res: header, username = res info = read_channel(self.url, cw) - p2f = get_p2f(cw) - if p2f: - self.single = False - videos = [Video(url) for url in info['urls']] - self.urls = [video.url for video in videos] - videos[0].url() - self.title = clean_title('[Channel] {}'.format(info['name'])) - self.setIcon(videos[0].thumb) - return - else: - cw.gal_num = self.url = info['urls'].pop(0) - if info['urls'] and cw.alive: - s = ', '.join(info['urls']) - self.exec_queue.put((s, 'downButton(customWidget)')) - video = Video(self.url) - video.url() - self.title = video.title + videos = [Video(url) for url in info['urls']] + video = self.process_playlist('[Channel] {}'.format(info['name']), videos) + else: + video = Video(self.url) + video.url() + self.title = video.title + self.setIcon(video.thumb) self.urls.append(video.url) diff --git a/src/extractor/yandere_downloader.py b/src/extractor/yandere_downloader.py index 682d905..e8471c6 100644 --- a/src/extractor/yandere_downloader.py +++ b/src/extractor/yandere_downloader.py @@ -1,10 +1,14 @@ -from bs4 import BeautifulSoup -from urllib.request import urlopen from urllib.parse import unquote -from utils import Downloader, urljoin, clean_title +from utils import Downloader, urljoin, clean_title, try_n from translator import tr_ import ree as re import os +import downloader + + +@try_n(4) +def read_soup(url): + return downloader.read_soup(url) @Downloader.register @@ -13,9 +17,6 @@ class Downloader_yandere(Downloader): URLS = ['yande.re'] MAX_CORE = 4 - def init(self): - pass - @classmethod def fix_url(cls, url): url = re.sub(r'\?page=[0-9]+&', '?', url) @@ -33,8 +34,7 @@ def read(self): ids = set() url = self.url while True: - html = urlopen(url) - soup = BeautifulSoup(html, "html.parser") + soup = read_soup(url) tmp = soup.find_all(attrs={'class':'directlink'}, href=True) for image_html in tmp: image_url = image_html['href'] diff --git a/src/extractor/youku_downloader.py b/src/extractor/youku_downloader.py index cf03f8f..8cd491b 100644 --- a/src/extractor/youku_downloader.py +++ b/src/extractor/youku_downloader.py @@ -11,9 +11,6 @@ class Downloader_youku(Downloader): type = 'youku' single = True URLS = ['v.youku.com'] - - def init(self): - self.url = self.url.replace('youku_', '', 1) def read(self): video = Video(self.url) diff --git a/src/extractor/youporn_downloader.py b/src/extractor/youporn_downloader.py index 7a51e89..8cd6db6 100644 --- a/src/extractor/youporn_downloader.py +++ b/src/extractor/youporn_downloader.py @@ -17,8 +17,8 @@ class Downloader_youporn(Downloader): display_name = 'YouPorn' def init(self): - if self.url.startswith('youporn_'): - self.url = 'https://www.youporn.com/watch/{}'.format(self.url.replace('youporn_', '', 1)) + if 'youporn.com' not in self.url.lower(): + self.url = 'https://www.youporn.com/watch/{}'.format(self.url) def read(self): video = Video(self.url) diff --git a/src/extractor/youtube_downloader.py b/src/extractor/youtube_downloader.py index 7291d8e..91dfbfc 100644 --- a/src/extractor/youtube_downloader.py +++ b/src/extractor/youtube_downloader.py @@ -7,7 +7,7 @@ from error_printer import print_error from timee import sleep import ree as re -from utils import urljoin, Downloader, Soup, try_n, get_print, filter_range, get_p2f, LazyUrl, query_url, compatstr, uuid, get_max_range, format_filename, clean_title, get_resolution, get_abr +from utils import urljoin, Downloader, Soup, try_n, get_print, filter_range, LazyUrl, query_url, compatstr, uuid, get_max_range, format_filename, clean_title, get_resolution, get_abr import ffmpeg import sys import constants @@ -331,9 +331,6 @@ class Downloader_youtube(Downloader): def init(self): ui_setting = self.ui_setting - if 'youtube_' in self.url: - self.url = u'https://www.youtube.com/watch?v={}'.format(self.url.replace('youtube_','')) - if self.customWidget.format: ext_result = self.customWidget.format else: @@ -348,6 +345,8 @@ def init(self): @classmethod def fix_url(cls, url): # 2033 + if not re.match('https?://.+', url, re.IGNORECASE): + url = 'https://www.youtube.com/watch?v={}'.format(url) qs = query_url(url) if 'v' in qs: url = url.split('?')[0] + '?v={}'.format(qs['v'][0]) @@ -379,27 +378,14 @@ def read(self): else: raise Exception('No videos') - if len(videos) > 1: - p2f = get_p2f(cw) - if p2f: - self.single = False - self.title = clean_title(info['title']) - self.urls = [video.url for video in videos] - video = videos[0] - self.setIcon(video.thumb) - return - else: - video = videos.pop(0) - cw.gal_num = cw.url = video.url._url - if videos and cw.alive: - s = u', '.join(video.url._url for video in videos) - self.exec_queue.put(([s, {'youtube':cw.format}], 'downButton(cw[0], format_selector=cw[1])')) - - self.urls.append(video.url) + if info['type'] != 'single': + video = self.process_playlist(info['title'], videos) + else: + self.urls.append(video.url) + self.title = video.title + self.artist = video.username self.setIcon(video.thumb) - - self.title = video.title def int_(x): @@ -417,15 +403,18 @@ def get_videos(url, type='video', only_mp4=False, audio_included=False, max_res= if '/channel/' in url or '/user/' in url or '/c/' in url: info = read_channel(url, n=n, cw=cw) + info['type'] = 'channel' info['title'] = u'[Channel] {}'.format(info['uploader']) if cw: info['urls'] = filter_range(info['urls'], cw.range) elif '/playlist' in url: info = read_playlist(url, n=n, cw=cw) + info['type'] = 'playlist' info['title'] = u'[Playlist] {}'.format(info['title']) if cw: info['urls'] = filter_range(info['urls'], cw.range) else: + info['type'] = 'single' info['urls'] = [url] info['videos'] = [Video(url, type, only_mp4, audio_included, max_res, max_abr, cw) for url in info['urls']]