3.0.0

application/lib/urlopener.py

@@ -89,7 +89,7 @@ def open(self, url, data=None, headers=None, timeout=None, method=None, **kwargs
         return self.patch_response(resp)
 
     #远程连接互联网的url
-    @UrlRetry(max_retries=2, delay=1, backoff=2)
+    @UrlRetry(max_retries=2, delay=2, backoff=2)
     def open_remote_url(self, url, data, headers, timeout, method, **kwargs):
         timeout = timeout if timeout else self.timeout
         headers = self.get_headers(url, headers)

docs/Chinese/extension.md

@@ -1,7 +1,7 @@
 ---
 sort: 4
 ---
-# Chrome/Edge浏览器扩展程序
+# 浏览器扩展程序
 
 
 
@@ -16,7 +16,11 @@ KindleEar提供一个Chrome/Edge浏览器扩展程序，除了提供bookmarklet
 
 ## 安装
 在Chrome/Edge应用商店搜索 "KindleEar"。   
-或者使用 [Chrome直达链接](https://chromewebstore.google.com/detail/kindleear/hjgdeckkpbdndigjkdlloacphoednmln)
+或者使用      
+[Chrome直达链接](https://chromewebstore.google.com/detail/kindleear/hjgdeckkpbdndigjkdlloacphoednmln)     
+
+[Edge直达链接](
+https://microsoftedge.microsoft.com/addons/detail/kbenhnoknpimfepkkngagppiebjgfokp)   
 
 
 

docs/English/extension.md

@@ -1,7 +1,7 @@
 ---
 sort: 4
 ---
-# Chrome/Edge Browser Extension
+# Browser Extension
 
 ## Overview
 KindleEar provides a Chrome/Edge browser extension that not only offers bookmarklet functionality but also includes a feature for generating web scraping recipes without writing any code. With this feature, you don't need to understand programming or write code; just a few clicks can generate a Recipe file that KindleEar can use.    
@@ -12,7 +12,13 @@ For many websites that heavily use JavaScript to dynamically generate pages (whe
 
 ## Installation
 Search for "KindleEar" in the Chrome/Edge Web Store.     
-Alternatively, here is [chrome direct link](https://chromewebstore.google.com/detail/kindleear/hjgdeckkpbdndigjkdlloacphoednmln).    
+Alternatively, here is     
+
+[chrome direct link](https://chromewebstore.google.com/detail/kindleear/hjgdeckkpbdndigjkdlloacphoednmln)     
+
+[Edge direct link](
+https://microsoftedge.microsoft.com/addons/detail/kbenhnoknpimfepkkngagppiebjgfokp)    
+
 
 
 

main.py

@@ -4,7 +4,7 @@
 # Visit <https://github.com/cdhigh/KindleEar> for the latest version
 # Author: cdhigh <https://github.com/cdhigh>
 
-__Version__ = '3.0.0d'
+__Version__ = '3.0.0'
 
 import os, sys, builtins, logging
 
@@ -26,6 +26,8 @@ def set_env():
         'SECRET_KEY', 'ADMIN_NAME', 'POCKET_CONSUMER_KEY', 'HIDE_MAIL_TO_LOCAL', 'LOG_LEVEL']
     for key in keys:
         cfgMap[key] = os.getenv(key) if key in os.environ else getattr(config, key)
+        if (key == 'APP_DOMAIN') and not cfgMap[key].startswith('http'):
+            cfgMap[key] = 'https://' + cfgMap[key]
         os.environ[key] = cfgMap[key]
     return cfgMap
 

readme.md

@@ -3,23 +3,23 @@ __English__ · [简体中文](readme_zh.md)
 ---
 
 # Announcement
-March 10, 2024    
-Official release of 3.0 beta, KindleEar is ready for deployment now.   
-The author has successfully deployed it to GAE/Docker/Oracle Cloud/PythonAnywhere.  
+May 1, 2024  **Official release of KindleEar 3**  
 
 **Significant Updates:**
-* Supports Python 3
-* Redesigned software framework
-* Support for multiple platforms, no longer limited to the GAE platform
-* Support for Calibre's recipe format directly
-* Preset with over 1,000 Calibre builtin recipe files
-* Built-in bilingual translation feature
+* Full support for Python 3
+* Redesigned software architecture
+* Cross-platform support, freeing you from dependence on GAE platform
+* Support for Calibre's recipe format without the need for modification
+* Built-in library of over a thousand Calibre recipe files
+* Integrated bilingual translation feature, breaking language barriers for effortless information retrieval and language learning
+* Built-in text-to-speech functionality, transforming daily news into audio for easy consumption without reading
+* Includes a browser extension, enabling effortless creation of web scraping recipe without coding, facilitating seamless content delivery from any website (brag)    
 
 
 
 # Introduction
 KindleEar is a web application which can be deployed on various Python-hosting platforms or VPS.   
-It automatically aggregates various web content into epub/mobi and delivers it to your Kindle or other e-book readers daily.   
+It automatically aggregates various web content into epub/mobi/mp3 and delivers it to your Kindle or other e-book readers daily.   
 
 
 ## The features included:
@@ -30,15 +30,16 @@ It automatically aggregates various web content into epub/mobi and delivers it t
 * Automatic daily scheduled push
 * Built-in sharing library, you can directly subscribe to feeds shared by other users, and you can also share your own feeds with others
 * Powerful and convenient email forwarding service
-* Integration with systems like Evernote/Pocket/Instapaper
+* Integration with systems like Evernote/Pocket/Instapaper/wallabag
 
 
 
 For other details, please refer to the **[project documentation](https://cdhigh.github.io/KindleEar)**.
 
 
 
-![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/scrshot.gif)
+
+![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/images/scrshot.gif)
 
 
 

readme_zh.md

@@ -2,23 +2,23 @@
 
 ---
 
-# 公告
-2024-03-10    
-正式发布3.0 beta版，代码库已经可以用来部署，作者已经将其成功部署到GAE/Docker/Oracle Cloud/pythonAnywhere。
+2024-05-01  **KindleEar3版本正式发布**     
 
 **主要新特性:**
-* 支持Python 3
-* 重新设计的软件架构
-* 多平台支持，不再受限于gae平台
-* 支持不用修改的Calibre的recipe格式
-* 预置Calibre的一千多个recipe文件
-* 内置双语对照翻译功能，打破语言障碍，获取信息和学习外语同步进行
+* 全面支持Python 3   
+* 全新设计的软件架构   
+* 跨平台支持，告别对 GAE 平台的依赖   
+* 支持 Calibre 的 recipe 格式，无需修改    
+* 内置一千多个 Calibre recipe 文件    
+* 内置双语对照翻译功能，突破语言壁垒，轻松获取信息和学习外语    
+* 内置文本转语音功能，将每日新闻转化为声音，让您无需阅读，也能轻松获取信息     
+* 包含浏览器扩展程序，无需编码即可制作爬虫脚本，便捷推送任意网站（虚假宣传）      
 
 
 
 # 简介
 这是一个Kindle个人推送服务应用，可以将其部署在各种支持Python的托管平台或VPS上。
-每天自动聚合各种网络信息制作成epub/mobi格式推送至您的Kindle或其他电子书阅读器。
+每天自动聚合各种网络信息制作成epub/mobi/mp3格式推送至您的Kindle或其他电子书阅读器。
 
 
 此应用目前的主要功能有：  
@@ -30,14 +30,15 @@
 * 自动每天定时推送
 * 内置共享库，可以直接订阅其他网友分享的订阅源，也可以分享自己的订阅源给其他网友
 * 强大而且方便的邮件中转服务
-* 和Evernote/Pocket/Instapaper等系统的集成
+* 和Evernote/Pocket/Instapaper/wallabag等系统的集成
 
 
 其他细节请参考 **[项目文档](https://cdhigh.github.io/KindleEar)**
 
 
 
-![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/scrshot.gif)
+
+![Screenshot](https://raw.githubusercontent.com/cdhigh/KindleEar/master/docs/images/scrshot.gif)
 
 
 

tests/readme.developer.md

@@ -31,7 +31,16 @@
 
 gcloud app deploy cron.yaml
 gcloud app deploy queue.yaml
+gcloud services list #current enabled services
 gcloud services list | grep datastore.googleapis.com
+gcloud services enable datastore.googleapis.com
+gcloud services enable tasks.googleapis.com
+gcloud services enable cloudtasks.googleapis.com
+gcloud services enable translate.googleapis.com
+gcloud services enable texttospeech.googleapis.com
+
+#all available services
+gcloud services list --available > services.txt
 
 # Windows 安装celery
 * 安装并启动redis服务，(Windows只能安装redis3 <https://github.com/MicrosoftArchive/redis/releases>)
@@ -58,9 +67,12 @@ gcloud services list | grep datastore.googleapis.com
 # 电子书简要生成流程
   build_ebook.ConvertToEbook() -> plumber.run() -> recipe_input.convert() -> news.BasicNewsRecipe.download()
   plumber.create_oebbook() -> OEBReader.call() -> output_plugin.convert()
+
 # KindleEar额外自带的Python库，这些库不用pip安装，不在requirements.txt里面
 * readability-lxml: 修改了其htmls.py|shorten_title()
 
+# 如果要添加新选项，最好添加到 calibre.customize.conversion.py | InputFormatPlugin | common_options, 
+
 # 关于i18n翻译
 * javascript的翻译没有采用其他复杂或引入其他依赖的方案，而是简单粗暴的在base.html里面将要翻译的字段预先翻译，
 然后保存到一个全局字典
@@ -77,8 +89,7 @@ tools\pybabel_compile.bat
 # Docker
 ## 构建镜像
 ```bash
-cp ./docker/Dockerfile .
-sudo docker build -t kindleear/kindleear .
+cd kindleear && cp ./docker/Dockerfile . && sudo docker build -t kindleear/kindleear .
 #or
 sudo docker build --no-cache -t kindleear/kindleear .
 sudo docker tag id kindleear/kindleear:version
@@ -104,17 +115,57 @@ sudo docker push kindleear/kindleear
 * sudo apt update && sudo apt install certbot
 * sudo certbot certonly --manual --preferred-challenges=dns --email [email protected] -d www.yourdomain.com
 * 添加txt记录
+* 自动续签方案：
+1. crontab
+```bash
+sudo crontab -e
+#打开编辑器后添加下面一行
+0 0 * * 1 /usr/bin/certbot renew >> /var/log/certbot-renew.log
+```
+
+2. systemd
+2.1 创建 /etc/systemd/system/certbot-renew.service
+```
+[Unit]
+Description=Renew SSL certificate with certbot
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/certbot renew
+ExecStartPost=/bin/cp -f /etc/letsencrypt/live/www.yourdomain.com/privkey.pem /home/ubuntu/data/privkey.pem
+ExecStartPost=/bin/cp -f /etc/letsencrypt/live/www.yourdomain.com/fullchain.pem /home/ubuntu/data/fullchain.pem
+```
+
+2.2 创建 /etc/systemd/system/certbot-renew.timer
+```
+[Unit]
+Description=Run certbot renew weekly
+[Timer]
+OnCalendar=Mon *-*-* 00:00:00
+Persistent=true
+[Install]
+WantedBy=timers.target
+```
+2.3 启用并启动定时器
+```bash
+sudo systemctl daemon-reload
+sudo systemctl enable certbot-renew.timer
+sudo systemctl start certbot-renew.timer
+```
+
+
 
 # Python托管平台的一些了解
 * [appengine](https://cloud.google.com)：必须绑定信用卡，但有免费额度，有收发邮件服务，任务队列，后台进程
 * [Heroku](https://www.heroku.com): 没有免费额度，入门套餐也需要付费
 * [Pythonanywhere](https://www.pythonanywhere.com): 有免费计划，不支持任务队列，有每天两次的预定任务计划
 * [Adaptable](https://adaptable.io): 免费计划不支持任何形式的任务队列和后台任务，5分钟内必须完成应答请求
 * [render](https://render.com): 有免费计划，没有免费cron额度
-* []()
+
+
 
 # 常用链接
 [App Engine 文档](https://cloud.google.com/appengine/docs)
 [yaml配置文档](https://cloud.google.com/appengine/docs/standard/reference/app-yaml?tab=python)
 [Cloud Tasks]https://cloud.google.com/tasks/docs
 [Adding your favorite news website](https://manual.calibre-ebook.com/news.html)
+[GAE限额](https://cloud.google.com/appengine/docs/standard/quotas)

tests/runtests.py

@@ -22,7 +22,8 @@ def set_env():
         'SECRET_KEY', 'ADMIN_NAME', 'POCKET_CONSUMER_KEY', 'HIDE_MAIL_TO_LOCAL']
     for key in keys:
         cfgMap[key] = os.getenv(key) if key in os.environ else getattr(config, key)
-        os.environ[key] = cfgMap[key]
+        if (key == 'APP_DOMAIN') and not cfgMap[key].startswith('http'):
+            cfgMap[key] = 'https://' + cfgMap[key]
     return cfgMap
 
 set_env()

tools/run_flask.bat

@@ -1,12 +1,14 @@
+mode con: cols=160 lines=1000
 D:
 cd D:\Programer\Project\KindleEar
 set FLASK_APP=main.py
 set APP_DOMAIN=http://localhost:5000/
 set DATABASE_URL=sqlite:///database.db
+::set DATABASE_URL=pickle:///database.pkl
 set TASK_QUEUE_SERVICE=apscheduler
 set TASK_QUEUE_BROKER_URL=memory
 set KE_TEMP_DIR=d:/temp
-set LOG_LEVEL=debug
+set LOG_LEVEL=info
 set HIDE_MAIL_TO_LOCAL=no
 python -m flask run --host=0.0.0.0 --debug
 pause

tools/update_req.py

@@ -228,33 +228,39 @@ def update_worker_yaml(workerYamlFile, arg):
             lines[idx] = f"  idle_timeout: {idle_timeout}"
         elif line.startswith('entrypoint:'):
             #entrypoint: gunicorn -b :$PORT --workers 1 --threads 2 --timeout 1200 main:app
-            parts = line.split(' ')
-            def elemIdx(e):
-                idx = parts.index(e) if e in parts else 99999
-                return idx if (idx < len(parts) - 1) else 0 #ensure have one more slot
+            parts = [item.strip() for item in line.split(' ') if item.strip()]
+            def elemIdx(key): #Find position of key
+                return parts.index(key) if key in parts else 0
 
-            wkIdx = elemIdx('--workers') or elemIdx('-w')
-            if wkIdx and parts[wkIdx + 1].isdigit() and max_instances:
-                parts[wkIdx + 1] = max_instances
-
-            tmIdx = elemIdx('--timeout') or elemIdx('-t')
-            if tmIdx and parts[tmIdx + 1].isdigit() and idle_timeout:
-                parts[tmIdx + 1] = str(int(int(idle_timeout[:-1]) * 60))
-
-            thIdx = elemIdx('--threads')
-            if thIdx and parts[thIdx + 1].isdigit() and threads:
-                parts[thIdx + 1] = threads
+            def UpdateOrAddParam(key1, value, key2=None): #Update if exists or add if not
+                idx = elemIdx(key1) or elemIdx(key2)
+                if idx > 0 and idx < len(parts) - 1 and parts[idx + 1].isdigit():
+                    parts[idx + 1] = value
+                elif 0 < idx < len(parts):
+                    parts.insert(idx + 1, value)
+                else:
+                    idx = len(parts) - 1
+                    parts[idx:idx] = [key1, value]
 
+            if max_instances:
+                UpdateOrAddParam('--workers', max_instances, '-w')
+            if idle_timeout:
+                UpdateOrAddParam('--timeout', str(int(int(idle_timeout[:-1]) * 60)), '-t')
+            if threads:
+                UpdateOrAddParam('--threads', threads)
+
             lines[idx] = ' '.join(parts)
 
-    with open(workerYamlFile, 'w', encoding='utf-8') as f:
-        f.write('\n'.join(lines))
-    print(f'Finished update of {workerYamlFile} using params:')
-    print(f'    instance_class: {instance_class}')
-    print(f'     max_instances: {max_instances}')
-    print(f'           threads: {threads}')
-    print(f'      idle_timeout: {idle_timeout}')
-    print('')
+    ret = [f'Finished update of {workerYamlFile} using params:']
+    instance_class and ret.append(f'    instance_class: {instance_class}')
+    max_instances and ret.append(f'     max_instances: {max_instances}')
+    threads and ret.append(f'           threads: {threads}')
+    idle_timeout and ret.append(f'      idle_timeout: {idle_timeout}')
+    ret.append('')
+    if len(ret) > 2:
+        with open(workerYamlFile, 'w', encoding='utf-8') as f:
+            f.write('\n'.join(lines))
+        print('\n'.join(ret))
 
 if __name__ == '__main__':
     thisDir = os.path.abspath(os.path.dirname(__file__))