From f6e8a657dfd3584cbf870470f1b19dc4edf54e92 Mon Sep 17 00:00:00 2001 From: Sean Feng Date: Thu, 31 Jan 2019 15:29:23 +0800 Subject: [PATCH] [add] shell mode [add] resources type choice [modify] code refactoring --- .vscode/launch.json | 110 +++------------- .vscode/settings.json | 3 +- ArtStationDownloader.py | 245 ------------------------------------ README.md | 20 ++- build.bat | 3 +- requirements.txt | 14 ++- src/ArtStationDownloader.py | 57 +++++++++ src/app.py | 126 +++++++++++++++++++ Config.py => src/config.py | 4 +- src/console.py | 12 ++ src/core.py | 135 ++++++++++++++++++++ 11 files changed, 381 insertions(+), 348 deletions(-) delete mode 100644 ArtStationDownloader.py create mode 100644 src/ArtStationDownloader.py create mode 100644 src/app.py rename Config.py => src/config.py (97%) create mode 100644 src/console.py create mode 100644 src/core.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 5d61493..469b0cb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,119 +1,45 @@ { - // 使用 IntelliSense 了解相关属性。 + // 使用 IntelliSense 了解相关属性。 // 悬停以查看现有属性的描述。 // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - { - "name": "Python: Current File", + "name": "Python: ArtStationDownloader", "type": "python", "request": "launch", - "program": "${file}", - "pythonPath": "${config:python.pythonPath}", - "envFile": "${workspaceRoot}/.env", - }, - { - "name": "Python: Attach", - "type": "python", - "request": "attach", - "localRoot": "${workspaceFolder}", - "remoteRoot": "${workspaceFolder}", - "port": 3000, - "secret": "my_secret", - "host": "localhost" - }, - { - "name": "Python: Terminal (integrated)", - "type": "python", - "request": "launch", - "program": "${file}", + "program": "${workspaceFolder}/src/ArtStationDownloader.py", "console": "integratedTerminal" }, { - "name": "Python: Terminal (external)", + "name": "Python: ArtStationDownload in Terminal", "type": "python", "request": "launch", - "program": "${file}", - "console": "externalTerminal" - }, - { - "name": "Python: Django", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/manage.py", + "program": "${workspaceFolder}/src/ArtStationDownloader.py", "args": [ - "runserver", - "--noreload", - "--nothreading" + "-u", + "ase", + "ikaired", + "-d", + "D:\\Users\\Sean\\Desktop\\ArtStation", + "-t", + "all" ], - "debugOptions": [ - "RedirectOutput", - "Django" - ] - }, - { - "name": "Python: Flask (0.11.x or later)", - "type": "python", - "request": "launch", - "module": "flask", - "env": { - "FLASK_APP": "${workspaceFolder}/app.py" - }, - "args": [ - "run", - "--no-debugger", - "--no-reload" - ] - }, - { - "name": "Python: Module", - "type": "python", - "request": "launch", - "module": "module.name" - }, - { - "name": "Python: Pyramid", - "type": "python", - "request": "launch", - "args": [ - "${workspaceFolder}/development.ini" - ], - "debugOptions": [ - "RedirectOutput", - "Pyramid" - ] + "console": "integratedTerminal" }, { - "name": "Python: Watson", + "name": "Python: Current File (Integrated Terminal)", "type": "python", "request": "launch", - "program": "${workspaceFolder}/console.py", - "args": [ - "dev", - "runserver", - "--noreload=True" - ] + "program": "${file}", + "console": "integratedTerminal" }, { - "name": "Python: All debug Options", + "name": "Python: Current File (External Terminal)", "type": "python", "request": "launch", - "pythonPath": "${config:python.pythonPath}", "program": "${file}", - "module": "module.name", - "env": { - "VAR1": "1", - "VAR2": "2" - }, - "envFile": "${workspaceFolder}/.env", - "args": [ - "arg1", - "arg2" - ], - "debugOptions": [ - "RedirectOutput" - ] + "console": "externalTerminal" } ] } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index ca42029..9b55de6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,4 @@ { - "python.pythonPath": "${workspaceFolder}\\.env\\Scripts\\python.exe" + "python.pythonPath": ".env\\Scripts\\python.exe", + "python.linting.pylintEnabled": true } \ No newline at end of file diff --git a/ArtStationDownloader.py b/ArtStationDownloader.py deleted file mode 100644 index 8f04143..0000000 --- a/ArtStationDownloader.py +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -"""批量下载ArtStation图片 - -Copyright 2018 Sean Feng(sean@FantaBlade.com) - -CHANGELOG - -20180611 0.1.0-alpha1 -允许在txt中使用Python风格的注释，即以#开头的内容会被忽略 -对txt中的空白符进行处理 -保存路径不再强制包含 ArtStation -默认保存路径现在为用户根路径 - -""" -__version__ = "0.1.0-alpha1" -# $Source$ - -from urllib.parse import urlparse -from concurrent import futures -from xml.dom.minidom import parseString - -from tkinter import Tk, Frame, Label, Button, Scrollbar, Text, Entry, messagebox, filedialog # 引入Tkinter工具包 -from tkinter import TOP, LEFT, BOTTOM, BOTH, X, Y, END -from tkinter import ttk - -import threading -import os -import requests -import json -import re -import pafy - -import Config - - -class App(Frame): - - def log(self, value): - self.text.configure(state="normal") - self.text.insert(END, value + '\n') - self.text.see(END) - self.text.configure(state="disabled") - - def download_file(self, url, file_path, file_name): - file_full_path = os.path.join(file_path, file_name) - if os.path.exists(file_full_path): - self.log('[Exist][image][{}]'.format(file_full_path)) - else: - r = requests.get(url) - os.makedirs(file_path, exist_ok=True) - with open(file_full_path, "wb") as code: - code.write(r.content) - self.log('[Finish][image][{}]'.format(file_full_path)) - - def download_video(self, id, file_path): - file_full_path = os.path.join(file_path, "{}.{}".format(id, 'mp4')) - if os.path.exists(file_full_path): - self.log('[Exist][video][{}]'.format(file_full_path)) - else: - video = pafy.new(id) - best = video.getbest(preftype="mp4") - r = requests.get(best.url) - os.makedirs(file_path, exist_ok=True) - with open(file_full_path, "wb") as code: - code.write(r.content) - self.log('[Exist][video][{}]'.format(file_full_path)) - - def download_project(self, hash_id): - url = 'https://www.artstation.com/projects/{}.json'.format(hash_id) - r = requests.get(url) - j = r.json() - assets = j['assets'] - title = j['slug'].strip() - # self.log('=========={}=========='.format(title)) - username = j['user']['username'] - for asset in assets: - file_path = os.path.join(self.root_path, username, title) - if asset['has_image']: # 包含图片 - url = asset['image_url'] - file_name = urlparse(url).path.split('/')[-1] - try: - self.executor.submit(self.download_file, - url, file_path, file_name) - except Exception as e: - print(e) - if asset['has_embedded_player']: # 包含视频 - player_embedded = asset['player_embedded'] - id = re.search( - r'(?<=https://www\.youtube\.com/embed/)[\w_]+', player_embedded).group() - self.executor_video.submit(self.download_video, id, file_path) - - def get_projects(self, username): - data = [] - if username is not '': - page = 0 - user_path = os.path.join(self.root_path, username) - while True: - page += 1 - url = 'https://www.artstation.com/users/{}/projects.json?page={}'.format( - username, page) - r = requests.get(url) - if not r.ok: - self.log("[Error] Please input right username") - break - j = r.json() - total_count = int(j['total_count']) - if total_count == 0: - self.log("[Error] Please input right username") - break - if page is 1: - self.log('=========={}=========='.format(username)) - os.makedirs(user_path, exist_ok=True) - data_fragment = j['data'] - data += data_fragment - self.log('==========Get page {}/{}=========='.format(page, - total_count // 50 + 1)) - if page > total_count / 50: - break - return data - - def download_by_usernames(self, usernames): - max_workers = 20 - if self.executor is None: - self.executor = futures.ThreadPoolExecutor(max_workers) - # 去重与处理网址 - username_set = set() - for username in usernames: - username = username.strip().split('/')[-1] - username_set.add(username) - for username in username_set: - data = self.get_projects(username) - if len(data) is not 0: - args = [project['hash_id'] for project in data] - for hash_id in args: - self.executor.submit(self.download_project, hash_id) - - def download(self): - self.btn_download.configure(state="disabled") - self.btn_download_txt.configure(state="disabled") - username_text = self.entry_filename.get() - if len(username_text) is 0: - messagebox.showinfo( - title='Warning', message='Please input usernames') - else: - usernames = username_text.split(',') - self.download_by_usernames(usernames) - self.btn_download.configure(state="normal") - self.btn_download_txt.configure(state="normal") - - def download_txt(self): - self.btn_download.configure(state="disabled") - self.btn_download_txt.configure(state="disabled") - filename = os.path.normpath(filedialog.askopenfilename( - filetypes=(('text files', '*.txt'), ("all files", "*.*")))) - if filename is not '.': - with open(filename, "r") as f: - usernames = [] - # 预处理，去掉注释与空白符 - for username in f.readlines(): - username = username.strip() - if len(username) is 0: - continue - sharp_at = username.find('#') - if sharp_at is 0: - continue - if sharp_at is not -1: - username = username[:sharp_at] - usernames.append(username.strip()) - self.download_by_usernames(usernames) - self.btn_download.configure(state="normal") - self.btn_download_txt.configure(state="normal") - - def browse_directory(self): - dir = os.path.normpath(filedialog.askdirectory()) - if dir is not '': - self.root_path = dir - Config.write_config('config.ini', 'Paths', - 'root_path', self.root_path) - self.entry_path.delete(0, END) - self.entry_path.insert(0, self.root_path) - - def createWidgets(self): - frame_tool = Frame(self.window) - frame_path = Frame(self.window) - frame_log = Frame(self.window) - self.lbl_username = Label( - frame_tool, text='Usernames(split by \',\'):') - self.entry_filename = Entry(frame_tool) - self.btn_download = Button( - frame_tool, text='Download', command=lambda: self.executor_ui.submit(self.download)) - self.btn_download_txt = Button( - frame_tool, text='Download txt', command=lambda: self.executor_ui.submit(self.download_txt)) - self.lbl_path = Label(frame_path, text='Path:') - self.entry_path = Entry(frame_path) - self.entry_path.insert(END, self.root_path) - self.btn_path_dialog = Button( - frame_path, text="Browse", command=lambda: self.browse_directory()) - self.scrollbar = Scrollbar(frame_log) - self.text = Text(frame_log) - self.text.configure(state="disabled") - self.lbl_status = Label( - self.window, text='Feel free to use! Support: Sean Feng(sean@FantaBlade.com)') - - frame_tool.pack(side=TOP, fill=X) - self.lbl_username.pack(side=LEFT) - self.entry_filename.pack(side=LEFT, fill=X, expand=True) - self.btn_download.pack(side=LEFT) - self.btn_download_txt.pack(side=LEFT) - frame_path.pack(side=TOP, fill=X) - self.lbl_path.pack(side=LEFT) - self.entry_path.pack(side=LEFT, fill=X, expand=True) - self.btn_path_dialog.pack(side=LEFT) - self.text.pack(side=LEFT, fill=BOTH, expand=True) - self.scrollbar.pack(side=LEFT, fill=Y) - frame_log.pack(side=TOP, fill=BOTH, expand=True) - self.scrollbar.config(command=self.text.yview) - self.text.config(yscrollcommand=self.scrollbar.set) - self.text.focus() - self.lbl_status.pack(side=LEFT, fill=X, expand=True) - - def __init__(self, master=None): - Frame.__init__(self, master) - master.title('ArtStation Downloader ' + __version__) # 定义窗体标题 - root_path_config = Config.read_config( - 'config.ini', 'Paths', 'root_path') - self.root_path = os.path.join( - os.path.expanduser("~")) if root_path_config is '' else root_path_config - self.executor = None - self.executor_ui = futures.ThreadPoolExecutor(1) - self.executor_video = futures.ThreadPoolExecutor(1) - self.window = master - self.pack() - self.createWidgets() - - -def main(): - window = Tk() - app = App(master=window) - app.mainloop() # 进入主循环，程序运行 - - -if __name__ == '__main__': - main() diff --git a/README.md b/README.md index 8cc44e7..2e74fb0 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ or you can create a txt file with These, one artist one line. and click the Download txt button to select file. +The combobox named Type means you can choose what resources are required. image only, video only or both + The Path means the path of the download folder，just set it. 输入你希望下载的作者的主页地址，或者其用户名 @@ -49,6 +51,8 @@ https://www.artstation.com/xrnothing 或者 xrnothing 然后点击 Download txt 按钮选择文件即可。 +Type 下拉选单可以设置下载资源的类型, 你可以选择“只下载图片”、“只下载视频”以及“全部下载” + Path 输入框内是下载文件夹位置，你可以按需求设置。 ## Bugs and Feedback @@ -67,8 +71,22 @@ Require Pyinstaller. Just execute build.bat +## For macOS/Linux/Shell + +install dependencies py run `pip install -r requirements.txt` first + +just run `python ./src/ArtStationDownloader.py` in shell to run in GUI mode + +or + +run like this: + +`python ./src/ArtStationDownloader.py -u username_of_artist other_username or_more -d where/you/what` in shell + +try `python ./src/ArtStationDownloader.py --help` to get more usage + ## LICENSE MIT License -Copyright (c) 2018 Sean Feng \ No newline at end of file +Copyright (c) 2018-2019 Sean Feng \ No newline at end of file diff --git a/build.bat b/build.bat index 23bef69..fe8ea11 100644 --- a/build.bat +++ b/build.bat @@ -1 +1,2 @@ -pyinstaller --windowed .\ArtStationDownloader.py \ No newline at end of file +pyinstaller --windowed .\src\ArtStationDownloader.py +@pause \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4329795..6c96eb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,18 @@ astroid==2.1.0 +autopep8==1.4.3 certifi==2018.11.29 chardet==3.0.4 colorama==0.4.1 -idna==2.7 +idna==2.8 isort==4.3.4 lazy-object-proxy==1.3.1 mccabe==0.6.1 pafy==0.5.4 +pycodestyle==2.4.0 pylint==2.2.2 -requests==2.20.1 -six==1.11.0W -typed-ast==1.1.0 +requests==2.21.0 +six==1.12.0 +typed-ast==1.3.0 urllib3==1.24.1 -wrapt==1.10.11 -youtube-dl==2018.11.23 +wrapt==1.11.1 +youtube-dl==2019.1.30.1 diff --git a/src/ArtStationDownloader.py b/src/ArtStationDownloader.py new file mode 100644 index 0000000..bcabeb6 --- /dev/null +++ b/src/ArtStationDownloader.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""批量下载ArtStation图片 + +Copyright 2018-2019 Sean Feng(sean@FantaBlade.com) + +CHANGELOG + +20180611 0.1.0-alpha1 +允许在txt中使用Python风格的注释，即以#开头的内容会被忽略 +对txt中的空白符进行处理 +保存路径不再强制包含 ArtStation +默认保存路径现在为用户根路径 + +20190131 0.1.1-alpha +重构代码结构 +加入命令行模式 +GUI和命令行均加入允许选择下载文件类型（所有、图片、视频）功能 +""" +__version__ = "0.1.1-alpha" +# $Source$ + +import argparse + +from app import App +from console import Console + + +def main(): + parser = argparse.ArgumentParser( + prog='ArtStationDownloader', + description='ArtStation Downloader is a lightweight tool to help you download images and videos from the ArtStation') + parser.add_argument('--version', action='version', + version='%(prog)s '+__version__) + parser.add_argument('-u', '--username', + help='choose who\'s project you want to download, one or more', nargs='*') + parser.add_argument('-d', '--directory', help='output directory') + parser.add_argument( + '-t', '--type', choices=['all', 'image', 'video'], default="all", help='what do you what to download, default is all') + parser.add_argument('-v', '--verbosity', action="count", + help="increase output verbosity") + args = parser.parse_args() + + if args.username: + if args.directory: + console = Console() + console.download_by_usernames(args.username, args.directory, args.type) + else: + print("no output directory, please use -d or --directory option to set") + else: + app = App(version=__version__) + app.mainloop() # 进入主循环，程序运行 + + +if __name__ == '__main__': + main() diff --git a/src/app.py b/src/app.py new file mode 100644 index 0000000..850b13e --- /dev/null +++ b/src/app.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- + +import os +from concurrent import futures + +from tkinter import Tk, Frame, Label, Button, Scrollbar, Text, Entry, messagebox, filedialog # 引入Tkinter工具包 +from tkinter import TOP, LEFT, BOTTOM, BOTH, X, Y, END +from tkinter import ttk + +import config +from core import Core + + +class App(Frame): + + def log(self, value): + self.text.configure(state="normal") + self.text.insert(END, value + '\n') + self.text.see(END) + self.text.configure(state="disabled") + + def download(self): + self.btn_download.configure(state="disabled") + self.btn_download_txt.configure(state="disabled") + username_text = self.entry_filename.get() + if len(username_text) is 0: + messagebox.showinfo( + title='Warning', message='Please input usernames') + else: + usernames = username_text.split(',') + self.core.root_path = self.root_path + self.core.download_by_usernames(usernames, self.combobox_type.current()) + self.btn_download.configure(state="normal") + self.btn_download_txt.configure(state="normal") + + def download_txt(self): + self.btn_download.configure(state="disabled") + self.btn_download_txt.configure(state="disabled") + filename = os.path.normpath(filedialog.askopenfilename( + filetypes=(('text files', '*.txt'), ("all files", "*.*")))) + if filename is not '.': + with open(filename, "r") as f: + usernames = [] + # 预处理，去掉注释与空白符 + for username in f.readlines(): + username = username.strip() + if len(username) is 0: + continue + sharp_at = username.find('#') + if sharp_at is 0: + continue + if sharp_at is not -1: + username = username[:sharp_at] + usernames.append(username.strip()) + self.core.root_path = self.root_path + self.core.download_by_usernames(usernames, self.combobox_type.current()) + self.btn_download.configure(state="normal") + self.btn_download_txt.configure(state="normal") + + def browse_directory(self): + dir = os.path.normpath(filedialog.askdirectory()) + if dir is not '': + self.root_path = dir + config.write_config('config.ini', 'Paths', + 'root_path', self.root_path) + self.entry_path.delete(0, END) + self.entry_path.insert(0, self.root_path) + + def createWidgets(self): + frame_tool = Frame(self.window) + frame_path = Frame(self.window) + frame_log = Frame(self.window) + self.lbl_username = Label( + frame_tool, text='Usernames(split by \',\'):') + self.entry_filename = Entry(frame_tool) + self.btn_download = Button( + frame_tool, text='Download', command=lambda: self.executor_ui.submit(self.download)) + self.btn_download_txt = Button( + frame_tool, text='Download txt', command=lambda: self.executor_ui.submit(self.download_txt)) + self.lbl_type = Label(frame_path, text='Type:') + self.combobox_type = ttk.Combobox(frame_path, state='readonly') + self.combobox_type["values"] = ('all', 'image', 'video') + self.combobox_type.current(0) + self.lbl_path = Label(frame_path, text='Path:') + self.entry_path = Entry(frame_path) + self.entry_path.insert(END, self.root_path) + self.btn_path_dialog = Button( + frame_path, text="Browse", command=lambda: self.browse_directory()) + self.scrollbar = Scrollbar(frame_log) + self.text = Text(frame_log) + self.text.configure(state="disabled") + self.lbl_status = Label( + self.window, text='Feel free to use! Support: Sean Feng(sean@FantaBlade.com)') + + frame_tool.pack(side=TOP, fill=X) + self.lbl_username.pack(side=LEFT) + self.entry_filename.pack(side=LEFT, fill=X, expand=True) + self.btn_download.pack(side=LEFT) + self.btn_download_txt.pack(side=LEFT) + frame_path.pack(side=TOP, fill=X) + self.lbl_type.pack(side=LEFT) + self.combobox_type.pack(side=LEFT) + self.lbl_path.pack(side=LEFT) + self.entry_path.pack(side=LEFT, fill=X, expand=True) + self.btn_path_dialog.pack(side=LEFT) + self.text.pack(side=LEFT, fill=BOTH, expand=True) + self.scrollbar.pack(side=LEFT, fill=Y) + frame_log.pack(side=TOP, fill=BOTH, expand=True) + self.scrollbar.config(command=self.text.yview) + self.text.config(yscrollcommand=self.scrollbar.set) + self.text.focus() + self.lbl_status.pack(side=LEFT, fill=X, expand=True) + + def __init__(self, version): + self.core = Core(self.log) + master = Tk() + Frame.__init__(self, master) + master.title('ArtStation Downloader ' + version) # 定义窗体标题 + root_path_config = config.read_config( + 'config.ini', 'Paths', 'root_path') + self.root_path = os.path.join( + os.path.expanduser("~"), "ArtStation") if root_path_config is '' else root_path_config + self.executor_ui = futures.ThreadPoolExecutor(1) + self.window = master + self.pack() + self.createWidgets() diff --git a/Config.py b/src/config.py similarity index 97% rename from Config.py rename to src/config.py index 1726c98..9df46a3 100644 --- a/Config.py +++ b/src/config.py @@ -1,5 +1,5 @@ -#!/usr/bin/python -# -*- coding:utf-8 -*- +#!/usr/bin/python +# -*- coding:utf-8 -*- # author: lingyue.wkl # desc: use to read ini # --------------------- diff --git a/src/console.py b/src/console.py new file mode 100644 index 0000000..aab261a --- /dev/null +++ b/src/console.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +from core import Core + +class Console: + + def __init__(self): + self.core = Core() + + def download_by_usernames(self, usernames, directory, type): + self.core.root_path = directory + self.core.download_by_usernames(usernames, type) diff --git a/src/core.py b/src/core.py new file mode 100644 index 0000000..bbd76e1 --- /dev/null +++ b/src/core.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- + +"""内核方法 +Copyright 2018-2019 Sean Feng(sean@FantaBlade.com) +""" + +import os +import re +from concurrent import futures +from multiprocessing import cpu_count +from urllib.parse import urlparse + +import pafy +import requests + + +class Core: + + def log(self, message): + print(message) + + def __init__(self, log_print=None): + if log_print: + global print + print = log_print + max_workers = cpu_count()*4 + self.executor = futures.ThreadPoolExecutor(max_workers) + self.executor_video = futures.ThreadPoolExecutor(1) + self.root_path = None + self.futures = [] + + def download_file(self, url, file_path, file_name): + file_full_path = os.path.join(file_path, file_name) + if os.path.exists(file_full_path): + self.log('[Exist][image][{}]'.format(file_full_path)) + else: + r = requests.get(url) + os.makedirs(file_path, exist_ok=True) + with open(file_full_path, "wb") as code: + code.write(r.content) + self.log('[Finish][image][{}]'.format(file_full_path)) + + def download_video(self, id, file_path): + file_full_path = os.path.join(file_path, "{}.{}".format(id, 'mp4')) + if os.path.exists(file_full_path): + self.log('[Exist][video][{}]'.format(file_full_path)) + else: + video = pafy.new(id) + best = video.getbest(preftype="mp4") + r = requests.get(best.url) + os.makedirs(file_path, exist_ok=True) + with open(file_full_path, "wb") as code: + code.write(r.content) + self.log('[Finish][video][{}]'.format(file_full_path)) + + def download_project(self, hash_id): + url = 'https://www.artstation.com/projects/{}.json'.format(hash_id) + r = requests.get(url) + j = r.json() + assets = j['assets'] + title = j['slug'].strip() + # self.log('=========={}=========='.format(title)) + username = j['user']['username'] + for asset in assets: + assert(self.root_path) + user_path = os.path.join(self.root_path, username) + os.makedirs(user_path, exist_ok=True) + file_path = os.path.join(user_path, title) + if not self.no_image and asset['has_image']: # 包含图片 + url = asset['image_url'] + file_name = urlparse(url).path.split('/')[-1] + try: + self.futures.append(self.executor.submit(self.download_file, + url, file_path, file_name)) + except Exception as e: + print(e) + if not self.no_video and asset['has_embedded_player']: # 包含视频 + player_embedded = asset['player_embedded'] + id = re.search( + r'(?<=https://www\.youtube\.com/embed/)[\w_]+', player_embedded).group() + try: + self.futures.append(self.executor_video.submit( + self.download_video, id, file_path)) + except Exception as e: + print(e) + + def get_projects(self, username): + data = [] + if username is not '': + page = 0 + while True: + page += 1 + url = 'https://www.artstation.com/users/{}/projects.json?page={}'.format( + username, page) + r = requests.get(url) + if not r.ok: + self.log("[Error] Please input right username") + break + j = r.json() + total_count = int(j['total_count']) + if total_count == 0: + self.log("[Error] Please input right username") + break + if page is 1: + self.log('\n==========[{}] BEGIN=========='.format(username)) + data_fragment = j['data'] + data += data_fragment + self.log('\n==========Get page {}/{}=========='.format(page, + total_count // 50 + 1)) + if page > total_count / 50: + break + return data + + def download_by_username(self, username): + data = self.get_projects(username) + if len(data) is not 0: + future_list = [] + for project in data: + future = self.executor.submit( + self.download_project, project['hash_id']) + future_list.append(future) + futures.wait(future_list) + + def download_by_usernames(self, usernames, type): + self.no_image = type == 'video' + self.no_video = type == 'image' + # 去重与处理网址 + username_set = set() + for username in usernames: + username = username.strip().split('/')[-1] + if username not in username_set: + username_set.add(username) + self.download_by_username(username) + futures.wait(self.futures) + self.log("\n========ALL DONE========")