diff --git a/application/lib/urlopener.py b/application/lib/urlopener.py index 7e411d7f..c467d861 100644 --- a/application/lib/urlopener.py +++ b/application/lib/urlopener.py @@ -89,7 +89,7 @@ def open(self, url, data=None, headers=None, timeout=None, method=None, **kwargs return self.patch_response(resp) #远程连接互联网的url - @UrlRetry(max_retries=2, delay=1, backoff=2) + @UrlRetry(max_retries=2, delay=2, backoff=2) def open_remote_url(self, url, data, headers, timeout, method, **kwargs): timeout = timeout if timeout else self.timeout headers = self.get_headers(url, headers) diff --git a/docs/Chinese/extension.md b/docs/Chinese/extension.md index 12c5fc08..5a9d2052 100644 --- a/docs/Chinese/extension.md +++ b/docs/Chinese/extension.md @@ -1,7 +1,7 @@ --- sort: 4 --- -# Chrome/Edge浏览器扩展程序 +# 浏览器扩展程序 @@ -16,7 +16,11 @@ KindleEar提供一个Chrome/Edge浏览器扩展程序，除了提供bookmarklet ## 安装 在Chrome/Edge应用商店搜索 "KindleEar"。 -或者使用 [Chrome直达链接](https://chromewebstore.google.com/detail/kindleear/hjgdeckkpbdndigjkdlloacphoednmln) +或者使用 +[Chrome直达链接](https://chromewebstore.google.com/detail/kindleear/hjgdeckkpbdndigjkdlloacphoednmln) + +[Edge直达链接]( +https://microsoftedge.microsoft.com/addons/detail/kbenhnoknpimfepkkngagppiebjgfokp) diff --git a/docs/English/extension.md b/docs/English/extension.md index 5d2f3b14..14873572 100644 --- a/docs/English/extension.md +++ b/docs/English/extension.md @@ -1,7 +1,7 @@ --- sort: 4 --- -# Chrome/Edge Browser Extension +# Browser Extension ## Overview KindleEar provides a Chrome/Edge browser extension that not only offers bookmarklet functionality but also includes a feature for generating web scraping recipes without writing any code. With this feature, you don't need to understand programming or write code; just a few clicks can generate a Recipe file that KindleEar can use. @@ -12,7 +12,13 @@ For many websites that heavily use JavaScript to dynamically generate pages (whe ## Installation Search for "KindleEar" in the Chrome/Edge Web Store. -Alternatively, here is [chrome direct link](https://chromewebstore.google.com/detail/kindleear/hjgdeckkpbdndigjkdlloacphoednmln). +Alternatively, here is + +[chrome direct link](https://chromewebstore.google.com/detail/kindleear/hjgdeckkpbdndigjkdlloacphoednmln) + +[Edge direct link]( +https://microsoftedge.microsoft.com/addons/detail/kbenhnoknpimfepkkngagppiebjgfokp) + diff --git a/docs/scrshot.gif b/docs/images/scrshot.gif similarity index 100% rename from docs/scrshot.gif rename to docs/images/scrshot.gif diff --git a/main.py b/main.py index 176b4c39..46e56a0a 100644 --- a/main.py +++ b/main.py @@ -4,7 +4,7 @@ # Visit for the latest version # Author: cdhigh -__Version__ = '3.0.0d' +__Version__ = '3.0.0' import os, sys, builtins, logging @@ -26,6 +26,8 @@ def set_env(): 'SECRET_KEY', 'ADMIN_NAME', 'POCKET_CONSUMER_KEY', 'HIDE_MAIL_TO_LOCAL', 'LOG_LEVEL'] for key in keys: cfgMap[key] = os.getenv(key) if key in os.environ else getattr(config, key) + if (key == 'APP_DOMAIN') and not cfgMap[key].startswith('http'): + cfgMap[key] = 'https://' + cfgMap[key] os.environ[key] = cfgMap[key] return cfgMap diff --git a/readme.md b/readme.md index b2eb3a17..c9fed26b 100644 --- a/readme.md +++ b/readme.md @@ -3,23 +3,23 @@ __English__ · [简体中文](readme_zh.md) --- # Announcement -March 10, 2024 -Official release of 3.0 beta, KindleEar is ready for deployment now. -The author has successfully deployed it to GAE/Docker/Oracle Cloud/PythonAnywhere. +May 1, 2024 **Official release of KindleEar 3** **Significant Updates:** -* Supports Python 3 -* Redesigned software framework -* Support for multiple platforms, no longer limited to the GAE platform -* Support for Calibre's recipe format directly -* Preset with over 1,000 Calibre builtin recipe files -* Built-in bilingual translation feature +* Full support for Python 3 +* Redesigned software architecture +* Cross-platform support, freeing you from dependence on GAE platform +* Support for Calibre's recipe format without the need for modification +* Built-in library of over a thousand Calibre recipe files +* Integrated bilingual translation feature, breaking language barriers for effortless information retrieval and language learning +* Built-in text-to-speech functionality, transforming daily news into audio for easy consumption without reading +* Includes a browser extension, enabling effortless creation of web scraping recipe without coding, facilitating seamless content delivery from any website (brag) # Introduction KindleEar is a web application which can be deployed on various Python-hosting platforms or VPS. -It automatically aggregates various web content into epub/mobi and delivers it to your Kindle or other e-book readers daily. +It automatically aggregates various web content into epub/mobi/mp3 and delivers it to your Kindle or other e-book readers daily. ## The features included: @@ -30,7 +30,7 @@ It automatically aggregates various web content into epub/mobi and delivers it t * Automatic daily scheduled push * Built-in sharing library, you can directly subscribe to feeds shared by other users, and you can also share your own feeds with others * Powerful and convenient email forwarding service -* Integration with systems like Evernote/Pocket/Instapaper +* Integration with systems like Evernote/Pocket/Instapaper/wallabag @@ -38,7 +38,8 @@ For other details, please refer to the **[project documentation](https://cdhigh. -![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/scrshot.gif) + +![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/images/scrshot.gif) diff --git a/readme_zh.md b/readme_zh.md index f5855290..50cbe141 100644 --- a/readme_zh.md +++ b/readme_zh.md @@ -2,23 +2,23 @@ --- -# 公告 -2024-03-10 -正式发布3.0 beta版，代码库已经可以用来部署，作者已经将其成功部署到GAE/Docker/Oracle Cloud/pythonAnywhere。 +2024-05-01 **KindleEar3版本正式发布** **主要新特性:** -* 支持Python 3 -* 重新设计的软件架构 -* 多平台支持，不再受限于gae平台 -* 支持不用修改的Calibre的recipe格式 -* 预置Calibre的一千多个recipe文件 -* 内置双语对照翻译功能，打破语言障碍，获取信息和学习外语同步进行 +* 全面支持Python 3 +* 全新设计的软件架构 +* 跨平台支持，告别对 GAE 平台的依赖 +* 支持 Calibre 的 recipe 格式，无需修改 +* 内置一千多个 Calibre recipe 文件 +* 内置双语对照翻译功能，突破语言壁垒，轻松获取信息和学习外语 +* 内置文本转语音功能，将每日新闻转化为声音，让您无需阅读，也能轻松获取信息 +* 包含浏览器扩展程序，无需编码即可制作爬虫脚本，便捷推送任意网站（虚假宣传） # 简介 这是一个Kindle个人推送服务应用，可以将其部署在各种支持Python的托管平台或VPS上。 -每天自动聚合各种网络信息制作成epub/mobi格式推送至您的Kindle或其他电子书阅读器。 +每天自动聚合各种网络信息制作成epub/mobi/mp3格式推送至您的Kindle或其他电子书阅读器。 此应用目前的主要功能有： @@ -30,14 +30,15 @@ * 自动每天定时推送 * 内置共享库，可以直接订阅其他网友分享的订阅源，也可以分享自己的订阅源给其他网友 * 强大而且方便的邮件中转服务 -* 和Evernote/Pocket/Instapaper等系统的集成 +* 和Evernote/Pocket/Instapaper/wallabag等系统的集成 其他细节请参考 **[项目文档](https://cdhigh.github.io/KindleEar)** -![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/scrshot.gif) + +![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/images/scrshot.gif) diff --git a/tests/readme.developer.md b/tests/readme.developer.md index 7c3efcc5..50236c94 100644 --- a/tests/readme.developer.md +++ b/tests/readme.developer.md @@ -31,7 +31,16 @@ gcloud app deploy cron.yaml gcloud app deploy queue.yaml +gcloud services list #current enabled services gcloud services list | grep datastore.googleapis.com +gcloud services enable datastore.googleapis.com +gcloud services enable tasks.googleapis.com +gcloud services enable cloudtasks.googleapis.com +gcloud services enable translate.googleapis.com +gcloud services enable texttospeech.googleapis.com + +#all available services +gcloud services list --available > services.txt # Windows 安装celery * 安装并启动redis服务，(Windows只能安装redis3 ) @@ -58,9 +67,12 @@ gcloud services list | grep datastore.googleapis.com # 电子书简要生成流程 build_ebook.ConvertToEbook() -> plumber.run() -> recipe_input.convert() -> news.BasicNewsRecipe.download() plumber.create_oebbook() -> OEBReader.call() -> output_plugin.convert() + # KindleEar额外自带的Python库，这些库不用pip安装，不在requirements.txt里面 * readability-lxml: 修改了其htmls.py|shorten_title() +# 如果要添加新选项，最好添加到 calibre.customize.conversion.py | InputFormatPlugin | common_options, + # 关于i18n翻译 * javascript的翻译没有采用其他复杂或引入其他依赖的方案，而是简单粗暴的在base.html里面将要翻译的字段预先翻译， 然后保存到一个全局字典 @@ -77,8 +89,7 @@ tools\pybabel_compile.bat # Docker ## 构建镜像 ```bash -cp ./docker/Dockerfile . -sudo docker build -t kindleear/kindleear . +cd kindleear && cp ./docker/Dockerfile . && sudo docker build -t kindleear/kindleear . #or sudo docker build --no-cache -t kindleear/kindleear . sudo docker tag id kindleear/kindleear:version @@ -104,6 +115,44 @@ sudo docker push kindleear/kindleear * sudo apt update && sudo apt install certbot * sudo certbot certonly --manual --preferred-challenges=dns --email xx@xx.com -d www.yourdomain.com * 添加txt记录 +* 自动续签方案： +1. crontab +```bash +sudo crontab -e +#打开编辑器后添加下面一行 +0 0 * * 1 /usr/bin/certbot renew >> /var/log/certbot-renew.log +``` + +2. systemd +2.1 创建 /etc/systemd/system/certbot-renew.service +``` +[Unit] +Description=Renew SSL certificate with certbot +[Service] +Type=oneshot +ExecStart=/usr/bin/certbot renew +ExecStartPost=/bin/cp -f /etc/letsencrypt/live/www.yourdomain.com/privkey.pem /home/ubuntu/data/privkey.pem +ExecStartPost=/bin/cp -f /etc/letsencrypt/live/www.yourdomain.com/fullchain.pem /home/ubuntu/data/fullchain.pem +``` + +2.2 创建 /etc/systemd/system/certbot-renew.timer +``` +[Unit] +Description=Run certbot renew weekly +[Timer] +OnCalendar=Mon *-*-* 00:00:00 +Persistent=true +[Install] +WantedBy=timers.target +``` +2.3 启用并启动定时器 +```bash +sudo systemctl daemon-reload +sudo systemctl enable certbot-renew.timer +sudo systemctl start certbot-renew.timer +``` + + # Python托管平台的一些了解 * [appengine](https://cloud.google.com)：必须绑定信用卡，但有免费额度，有收发邮件服务，任务队列，后台进程 @@ -111,10 +160,12 @@ sudo docker push kindleear/kindleear * [Pythonanywhere](https://www.pythonanywhere.com): 有免费计划，不支持任务队列，有每天两次的预定任务计划 * [Adaptable](https://adaptable.io): 免费计划不支持任何形式的任务队列和后台任务，5分钟内必须完成应答请求 * [render](https://render.com): 有免费计划，没有免费cron额度 -* []() + + # 常用链接 [App Engine 文档](https://cloud.google.com/appengine/docs) [yaml配置文档](https://cloud.google.com/appengine/docs/standard/reference/app-yaml?tab=python) [Cloud Tasks]https://cloud.google.com/tasks/docs [Adding your favorite news website](https://manual.calibre-ebook.com/news.html) +[GAE限额](https://cloud.google.com/appengine/docs/standard/quotas) \ No newline at end of file diff --git a/tests/runtests.py b/tests/runtests.py index f4eab30c..d8d320f9 100644 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -22,7 +22,8 @@ def set_env(): 'SECRET_KEY', 'ADMIN_NAME', 'POCKET_CONSUMER_KEY', 'HIDE_MAIL_TO_LOCAL'] for key in keys: cfgMap[key] = os.getenv(key) if key in os.environ else getattr(config, key) - os.environ[key] = cfgMap[key] + if (key == 'APP_DOMAIN') and not cfgMap[key].startswith('http'): + cfgMap[key] = 'https://' + cfgMap[key] return cfgMap set_env() diff --git a/tools/run_flask.bat b/tools/run_flask.bat index 20e6d558..7dd74e62 100644 --- a/tools/run_flask.bat +++ b/tools/run_flask.bat @@ -1,12 +1,14 @@ +mode con: cols=160 lines=1000 D: cd D:\Programer\Project\KindleEar set FLASK_APP=main.py set APP_DOMAIN=http://localhost:5000/ set DATABASE_URL=sqlite:///database.db +::set DATABASE_URL=pickle:///database.pkl set TASK_QUEUE_SERVICE=apscheduler set TASK_QUEUE_BROKER_URL=memory set KE_TEMP_DIR=d:/temp -set LOG_LEVEL=debug +set LOG_LEVEL=info set HIDE_MAIL_TO_LOCAL=no python -m flask run --host=0.0.0.0 --debug pause diff --git a/tools/update_req.py b/tools/update_req.py index e6227d17..d448d692 100644 --- a/tools/update_req.py +++ b/tools/update_req.py @@ -228,33 +228,39 @@ def update_worker_yaml(workerYamlFile, arg): lines[idx] = f" idle_timeout: {idle_timeout}" elif line.startswith('entrypoint:'): #entrypoint: gunicorn -b :$PORT --workers 1 --threads 2 --timeout 1200 main:app - parts = line.split(' ') - def elemIdx(e): - idx = parts.index(e) if e in parts else 99999 - return idx if (idx < len(parts) - 1) else 0 #ensure have one more slot + parts = [item.strip() for item in line.split(' ') if item.strip()] + def elemIdx(key): #Find position of key + return parts.index(key) if key in parts else 0 - wkIdx = elemIdx('--workers') or elemIdx('-w') - if wkIdx and parts[wkIdx + 1].isdigit() and max_instances: - parts[wkIdx + 1] = max_instances - - tmIdx = elemIdx('--timeout') or elemIdx('-t') - if tmIdx and parts[tmIdx + 1].isdigit() and idle_timeout: - parts[tmIdx + 1] = str(int(int(idle_timeout[:-1]) * 60)) - - thIdx = elemIdx('--threads') - if thIdx and parts[thIdx + 1].isdigit() and threads: - parts[thIdx + 1] = threads + def UpdateOrAddParam(key1, value, key2=None): #Update if exists or add if not + idx = elemIdx(key1) or elemIdx(key2) + if idx > 0 and idx < len(parts) - 1 and parts[idx + 1].isdigit(): + parts[idx + 1] = value + elif 0 < idx < len(parts): + parts.insert(idx + 1, value) + else: + idx = len(parts) - 1 + parts[idx:idx] = [key1, value] + if max_instances: + UpdateOrAddParam('--workers', max_instances, '-w') + if idle_timeout: + UpdateOrAddParam('--timeout', str(int(int(idle_timeout[:-1]) * 60)), '-t') + if threads: + UpdateOrAddParam('--threads', threads) + lines[idx] = ' '.join(parts) - with open(workerYamlFile, 'w', encoding='utf-8') as f: - f.write('\n'.join(lines)) - print(f'Finished update of {workerYamlFile} using params:') - print(f' instance_class: {instance_class}') - print(f' max_instances: {max_instances}') - print(f' threads: {threads}') - print(f' idle_timeout: {idle_timeout}') - print('') + ret = [f'Finished update of {workerYamlFile} using params:'] + instance_class and ret.append(f' instance_class: {instance_class}') + max_instances and ret.append(f' max_instances: {max_instances}') + threads and ret.append(f' threads: {threads}') + idle_timeout and ret.append(f' idle_timeout: {idle_timeout}') + ret.append('') + if len(ret) > 2: + with open(workerYamlFile, 'w', encoding='utf-8') as f: + f.write('\n'.join(lines)) + print('\n'.join(ret)) if __name__ == '__main__': thisDir = os.path.abspath(os.path.dirname(__file__))