From 80438360096fb2a5138d73104d8d6960b9d04cd6 Mon Sep 17 00:00:00 2001 From: "Shiki@Z" Date: Sat, 22 Apr 2023 18:13:19 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E9=A1=B9=E7=9B=AE=E4=B8=BB?= =?UTF-8?q?=E4=BD=93=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitattributes | 2 + .gitignore | 138 +++++++++++++++++++++++++++++++ LICENSE | 21 +++++ README-en.md | 188 +++++++++++++++++++++++++++++++++++++++++++ README.md | 192 ++++++++++++++++++++++++++++++++++++++++++++ conf/push.py | 23 ++++++ conf/sql.py | 8 ++ conf/subscribers.py | 36 +++++++++ conf/tasks.py | 39 +++++++++ core/parse.py | 27 +++++++ core/sql.py | 90 +++++++++++++++++++++ core/task.py | 156 +++++++++++++++++++++++++++++++++++ main.py | 23 ++++++ push/base.py | 62 ++++++++++++++ push/mirai.py | 86 ++++++++++++++++++++ push/smtp.py | 52 ++++++++++++ requirements.txt | Bin 0 -> 202 bytes 17 files changed, 1143 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README-en.md create mode 100644 README.md create mode 100644 conf/push.py create mode 100644 conf/sql.py create mode 100644 conf/subscribers.py create mode 100644 conf/tasks.py create mode 100644 core/parse.py create mode 100644 core/sql.py create mode 100644 core/task.py create mode 100644 main.py create mode 100644 push/base.py create mode 100644 push/mirai.py create mode 100644 push/smtp.py create mode 100644 requirements.txt diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c586011 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +README-en.md linguist-language=English +README.md linguist-language=Chinese diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a81c8ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..97ec565 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Shiki@Z + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README-en.md b/README-en.md new file mode 100644 index 0000000..2cf8b0c --- /dev/null +++ b/README-en.md @@ -0,0 +1,188 @@ +# InspectionLite +InspectionLite is a Python project used to detect updates on specified web pages. It can automatically detect updates on the specified page based on the user-defined task list and push update notifications to specified users. + +# Features +- **Supports multiple notification methods**: This program supports multiple notification methods, such as QQ bot based on Mirai HTTP, SMTP emails, etc., to meet the different needs of users. +- **Customizable tasks**: InspectionLite supports reading the task list from the configuration file. Users can add new detection tasks according to their needs. +- **Easy to extend**: The program adopts a modular design, and users can easily write new notification methods and register them in the system. +- **High degree of customization**: Users can configure parameters according to their own needs, such as modifying database connection information, task lists, notification methods, etc. +- **High reliability**: The program runs stably and reliably, and can promptly send emails to administrators in case of abnormal situations. + +# Installation +1. Clone the repository to your local machine. +2. Enter the project directory. +3. Install dependencies: `pip install -r requirements.txt` +4. Configure program parameters: edit the configuration file in the `/conf` directory and modify parameter values as needed. +5. Run the program: `python main.py`. + +# Usage Examples +
+ 1. Modify the conf/push.py file + +```python +push_setting = { + # The administrator's email address. If there are any notification errors during runtime, they will be sent to this email address. + 'admin': { + 'email': 'admin@example.com' + }, + # Fill in the parameters for the Mirai notification protocol here. + # If you don't need Mirai notification, leave it blank. + # Note: This project defaults to using Mirai HTTP protocol for notification. + 'mirai': { + 'host': 'mirai.host.com:port', + 'verify_key': 'Mirai HTTP verify key', + # Fill in the sender's QQ number here. The sender must have logged in to Mirai Console. + 'sender': 10000 + }, + # Fill in the parameters for the SMTP (email) notification protocol here. + # This is required because messages will be sent via email in case of notification errors. + 'smtp': { + 'host': 'smtp.example.com', + 'port': 465, + 'username': 'inspectionlite@example.com', + 'password': 'example password' + } +} +``` + +
+ +
+ 2. Modify the conf/sql.py file + +```python +# The project uses a MySQL database, so fill in the login parameters for the database here. +# Note: The database user must have read/write permission for the database. +sql_setting = { + 'host': 'database address', + 'user': 'database username', + 'password': 'database password', + 'db': 'database to use' +} +``` + +
+ +
+ 3. Modify the conf/subscribers.py file + +```python +subscribers = [ + # User 1 + { + # If subscribing to all notifications, set range to True; + # If subscribing to some notifications, write them as a list here; + # If subscribing to no notifications, set range to False (or delete this user). + 'range': True, + 'mirai': { + # Enable Mirai notification by setting enable to True. + # If not using Mirai notification, ignore id and is_group. + 'enable': True, + 'id': 10001, + 'is_group': False, + }, + 'smtp': { + # Enable SMTP (email) notification by setting enable to True. + # If not using SMTP notification, ignore email. + 'enable': True, + 'email': 'user@example.com' + } + }, + # User 2 + { + 'range': ['CSP notifications'], + 'mirai': { + 'enable': True, + 'id': 20001, + 'is_group': True, + }, + 'smtp': { + 'enable': False + } + } +] +``` + +
+ +
+ 4. Modify the conf/tasks.py file + +```python +task_list = [ + 'CSP notifications', + 'NJ Tech Academic Affairs Office', +# Add your tasks here. +# The following two functions are the main process of the task. Please read the examples carefully and fill in the same format. +] + + +def get_task_url(task_name: str) -> str: + """ + Get the URL of the specified task. + + :param task_name: Task name + :return: URL + """ + if task_name == task_list[0]: + jump_request = task_url_parse('https://www.cspro.org/cms/show.action?code=jumpchanneltemplate') + return urljoin('https://www.cspro.org', findall('\".*\"', jump_request.script.text)[1].strip('\"')) + elif task_name == task_list[1]: + return 'http://jwc.njtech.edu.cn/' + + +def get_task_css(task_name: str) -> str: + """ + Get the CSS selector of the specified task. + + :param task_name: Task name + :return: CSS selector + """ + if task_name == task_list[0]: + return 'body > div.l_mainouter > div > div:nth-child(1) > div.l_newsmsg.clearfix > div.l_overflowhidden > ' \ + 'span > a ' + elif task_name == task_list[1]: + return '#notice > div.ct > ul > li > p > a' +``` + +
+ +
+ (Optional) 5. Add your own notification method under the push directory + +```python +# To implement your own notification method, you must inherit the push.base.PushBase class. +# Then, you only need to implement the _send_update method. +from push.base import PushBase + + +class MyPush(PushBase): + def _send_update(self, + task: str, + title: str, + url: str, + subscriber: dict) -> bool: + # Implement your notification method here. + # Suppose you use mypush.py as your file name here: + # 1. Put the global parameters you need to use in push_setting['mypush'] in conf/push.py (if necessary); + # 2. Put individual parameters for each user in subscribers[index]['mypush'] in conf/subscribers.py. + pass +``` + +
+ +# Technical Details +### The InspectionLite project mainly consists of the following modules: + +- `main.py`: Program entry point, responsible for creating database connections and task units. +- `/conf` directory: Stores the program's configuration files, such as MySQL connection information, notification methods, and task lists. +- `/core` directory: Stores the core code of the program, such as task detection, data storage, and notification function implementations. +- `/push` directory: Stores specific implementations of different notification methods. + +### The workflow of InspectionLite is as follows: + +- Obtain the Web page URL and CSS selector to be detected based on the task list. +- Detect the update status of the Web page, and if an update occurs, save the update record to the MySQL database. +- Use multiple notification methods to send update notifications to specified users. +- If a notification failure occurs, retry and email the administrator with details if the retry fails. +- The entire program runs on the main program and can automatically perform task detection and notification operations. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..3bf6ea0 --- /dev/null +++ b/README.md @@ -0,0 +1,192 @@ +# InspectionLite +InspectionLite 是一个用于检测指定 Web 页面更新的 Python 项目。它能够根据用户定义的任务列表，自动地检测指定页面的更新情况，并将更新信息推送给指定的用户。 + +[*English Version*](https://github.com/kressety/InspectionLite/blob/main/README-en.md) + +# 特点 +- **支持多种推送方式**：该程序支持多种推送方式，如 ~~企业微信、钉钉、~~ (待实现) 基于Mirai HTTP的QQ机器人和SMTP邮件等，能够满足不同用户的推送需求。 +- **可自定义任务**：InspectionLite支持从配置文件中读取任务列表，用户可以根据自己的需求添加新的检测任务。 +- **易于扩展**：程序采用模块化设计，用户可以方便地编写新的推送方法并注册到系统中。 +- **定制化程度高**：用户可以根据自己的需求进行相应的配置，如修改数据库连接信息、任务列表、推送方式等。 +- **可靠性强**：程序运行稳定可靠，能够在出现异常情况时及时发送邮件告知管理员。 + +# 安装 +1. 克隆仓库到本地 +2. 进入项目目录 +3. 安装依赖：`pip install -r requirements.txt` +4. 配置程序参数：编辑`/conf`目录下的配置文件，根据需要修改参数值。 +5. 运行程序：`python main.py` + +# 使用示例 +
+ 1. 修改 conf/push.py 文件 + +```python +push_setting = { + # 管理员邮箱，如果运行中出现推送错误则会发送到这个邮箱。 + 'admin': { + 'email': 'admin@example.com' + }, + # Mirai推送协议参数在这里填写， + # 如果不需要Mirai推送则不需要填写。 + # 注意：本项目默认通过Mirai HTTP协议推送。 + 'mirai': { + 'host': 'mirai.host.com:port', + 'verify_key': 'Mirai HTTP verify key', + # 这里填写消息发送者的QQ号，必须已经在Mirai Console上登录 + 'sender': 10000 + }, + # SMTP（电子邮箱）推送协议参数在这里填写。 + # 这里必须填写，因为出现推送错误会通过邮箱发送消息。 + 'smtp': { + 'host': 'smtp.example.com', + 'port': 465, + 'username': 'inspectionlite@example.com', + 'password': 'example password' + } +} +``` + +
+ +
+ 2. 修改 conf/sql.py 文件 + +```python +# 项目使用MySQL数据库，因此在这里填写数据库的登录参数。 +# 注意：数据库用户必须拥有该数据库的读/写权限。 +sql_setting = { + 'host': '数据库地址', + 'user': '数据库用户名', + 'password': '数据库密码', + 'db': '要使用的数据库' +} +``` + +
+ +
+ 3. 修改 conf/subscribers.py 文件 + +```python +subscribers = [ + # 这里是 1 号用户 + { + # 订阅的推送，如果全部订阅则将 range 设置为 True； + # 如果部分订阅，则将订阅的内容以 list 的形式写在这里； + # 如果不订阅任何内容，则将 range 置为 False （或者你可以直接删除这个用户）。 + 'range': True, + 'mirai': { + # 启用则将 enable 置为 True。 + 'enable': True, + # 如果不启用 Mirai 推送，则 id 和 is_group 的值都会被忽略。 + 'id': 10001, + # 如果推送目标是一个QQ群，则将 is_group 置为 True； + # 如果推送给单个用户，则此项为 False。 + 'is_group': False, + }, + 'smtp': { + # 同上，启用则置为 True。 + 'enable': True, + # 如果启用，则填写 email，否则此项会被忽略，不需要填写。 + 'email': 'user@example.com' + } + }, + # 这里是 2 号用户 + { + 'range': ['CSP通知'], + 'mirai': { + 'enable': True, + 'id': 20001, + 'is_group': True, + }, + 'smtp': { + 'enable': False + } + } +] +``` + +
+ +
+ 4. 修改 conf/tasks.py 文件 + +```python +task_list = [ + 'CSP通知', + '工大教务处', +# 将你的任务添加在这里。 +# 下面两个函数是任务的主要流程，请仔细阅读两个示例，并按照相同格式填写。 +] + + +def get_task_url(task_name: str) -> str: + """ + 获取指定任务的URL。 + + :param task_name: 任务名 + :return: URL + """ + if task_name == task_list[0]: + jump_request = task_url_parse('https://www.cspro.org/cms/show.action?code=jumpchanneltemplate') + return urljoin('https://www.cspro.org', findall('\".*\"', jump_request.script.text)[1].strip('\"')) + elif task_name == task_list[1]: + return 'http://jwc.njtech.edu.cn/' + + +def get_task_css(task_name: str) -> str: + """ + 获取指定任务的CSS选择器。 + + :param task_name: 任务名 + :return: CSS选择器 + """ + if task_name == task_list[0]: + return 'body > div.l_mainouter > div > div:nth-child(1) > div.l_newsmsg.clearfix > div.l_overflowhidden > ' \ + 'span > a ' + elif task_name == task_list[1]: + return '#notice > div.ct > ul > li > p > a' +``` + +
+ +
+ (可选) 5. 在 push 目录下添加自己的推送方式 + +```python +# 要实现自己的推送方式，必须继承 push.base.PushBase 类。 +# 然后，你只需要实现 _send_update 方法即可。 +from push.base import PushBase + + +class MyPush(PushBase): + def _send_update(self, + task: str, + title: str, + url: str, + subscriber: dict) -> bool: + # 在这里实现你的推送方法。 + # 假设在这里你使用了mypush.py作为你的文件名，那么： + # 1. 在conf/push.py的push_setting['mypush']中放置你需要使用的全局参数（如有必要）； + # 2. 在conf/subscribers.py的subscribers[index]['mypush']中放置每个用户的个人参数。 + pass +``` + +
+ +# 技术细节 +### InspectionLite 项目主要包含以下几个模块： + +- `main.py`：程序入口，负责创建数据库连接和任务单元。 +- `/conf`目录：存放程序的配置文件，如MySQL连接信息、推送方式和任务列表等。 +- `/core`目录：存放程序的核心代码，如任务检测、数据存储和推送功能的实现。 +- `/push`目录：存放不同的推送方式的具体实现。 + +### InspectionLite 的工作流程如下： + +- 根据任务列表获取需要检测的 Web 页面 URL 和 CSS 选择器。 +- 检测 Web 页面更新情况，若发生更新则将更新记录保存到 MySQL 数据库中。 +- 使用多种推送方式向指定用户发送更新通知。 +- 如果出现推送失败的情况，则进行重试，并在重试失败时向管理员发送邮件告知具体情况。 +- 整个程序运行在主程序上，可自动执行任务检测和推送操作。 \ No newline at end of file diff --git a/conf/push.py b/conf/push.py new file mode 100644 index 0000000..4138fe4 --- /dev/null +++ b/conf/push.py @@ -0,0 +1,23 @@ +push_setting = { + # 管理员邮箱，如果运行中出现推送错误则会发送到这个邮箱。 + 'admin': { + 'email': 'admin@example.com' + }, + # Mirai推送协议参数在这里填写， + # 如果不需要Mirai推送则不需要填写。 + # 注意：本项目默认通过Mirai HTTP协议推送。 + 'mirai': { + 'host': 'mirai.host.com:port', + 'verify_key': 'Mirai HTTP verify key', + # 这里填写消息发送者的QQ号，必须已经在Mirai Console上登录 + 'sender': 10000 + }, + # SMTP（电子邮箱）推送协议参数在这里填写。 + # 这里必须填写，因为出现推送错误会通过邮箱发送消息。 + 'smtp': { + 'host': 'smtp.example.com', + 'port': 465, + 'username': 'inspectionlite@example.com', + 'password': 'example password' + } +} \ No newline at end of file diff --git a/conf/sql.py b/conf/sql.py new file mode 100644 index 0000000..be9911f --- /dev/null +++ b/conf/sql.py @@ -0,0 +1,8 @@ +# 项目使用MySQL数据库，因此在这里填写数据库的登录参数。 +# 注意：数据库用户必须拥有该数据库的读/写权限。 +sql_setting = { + 'host': '数据库地址', + 'user': '数据库用户名', + 'password': '数据库密码', + 'db': '要使用的数据库' +} \ No newline at end of file diff --git a/conf/subscribers.py b/conf/subscribers.py new file mode 100644 index 0000000..c0a2a7d --- /dev/null +++ b/conf/subscribers.py @@ -0,0 +1,36 @@ +subscribers = [ + # 这里是 1 号用户 + { + # 订阅的推送，如果全部订阅则将 range 设置为 True； + # 如果部分订阅，则将订阅的内容以 list 的形式写在这里； + # 如果不订阅任何内容，则将 range 置为 False （或者你可以直接删除这个用户）。 + 'range': True, + 'mirai': { + # 启用则将 enable 置为 True。 + 'enable': True, + # 如果不启用 Mirai 推送，则 id 和 is_group 的值都会被忽略。 + 'id': 10001, + # 如果推送目标是一个QQ群，则将 is_group 置为 True； + # 如果推送给单个用户，则此项为 False。 + 'is_group': False, + }, + 'smtp': { + # 同上，启用则置为 True。 + 'enable': True, + # 如果启用，则填写 email，否则此项会被忽略，不需要填写。 + 'email': 'user@example.com' + } + }, + # 这里是 2 号用户 + { + 'range': ['CSP通知'], + 'mirai': { + 'enable': True, + 'id': 20001, + 'is_group': True, + }, + 'smtp': { + 'enable': False + } + } +] \ No newline at end of file diff --git a/conf/tasks.py b/conf/tasks.py new file mode 100644 index 0000000..3a63554 --- /dev/null +++ b/conf/tasks.py @@ -0,0 +1,39 @@ +from urllib.parse import urljoin + +from core.parse import task_url_parse +from re import findall + +task_list = [ + 'CSP通知', + '工大教务处', +# 将你的任务添加在这里。 +# 下面两个函数是任务的主要流程，请仔细阅读两个示例，并按照相同格式填写。 +] + + +def get_task_url(task_name: str) -> str: + """ + 获取指定任务的URL。 + + :param task_name: 任务名 + :return: URL + """ + if task_name == task_list[0]: + jump_request = task_url_parse('https://www.cspro.org/cms/show.action?code=jumpchanneltemplate') + return urljoin('https://www.cspro.org', findall('\".*\"', jump_request.script.text)[1].strip('\"')) + elif task_name == task_list[1]: + return 'http://jwc.njtech.edu.cn/' + + +def get_task_css(task_name: str) -> str: + """ + 获取指定任务的CSS选择器。 + + :param task_name: 任务名 + :return: CSS选择器 + """ + if task_name == task_list[0]: + return 'body > div.l_mainouter > div > div:nth-child(1) > div.l_newsmsg.clearfix > div.l_overflowhidden > ' \ + 'span > a ' + elif task_name == task_list[1]: + return '#notice > div.ct > ul > li > p > a' diff --git a/core/parse.py b/core/parse.py new file mode 100644 index 0000000..722ba87 --- /dev/null +++ b/core/parse.py @@ -0,0 +1,27 @@ +from typing import Union + +from bs4 import BeautifulSoup +from requests import get, RequestException + + +def task_url_parse(url: str) -> Union[BeautifulSoup, bool]: + """ + 解析任务URL，并返回BeautifulSoup对象。 + + :param url: 任务URL + :return: 如果完成解析，返回BS对象，否则返回False + """ + header = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/103.0.5060.134 Safari/537.36 Edg/103.0.1264.77 ' + } + try: + request = get(url, headers=header) + request.encoding = 'UTF-8' + if request.status_code == 200: + html_data = BeautifulSoup(request.text, features='html.parser') + return html_data + else: + return False + except RequestException: + return False diff --git a/core/sql.py b/core/sql.py new file mode 100644 index 0000000..085003e --- /dev/null +++ b/core/sql.py @@ -0,0 +1,90 @@ +from pymysql import connect, Error + + +def set_task_sql_connect( + host: str, + user: str, + password: str, + db: str) -> connect: + """ + 建立数据库连接。 + + :param host: 数据库地址 + :param user: 数据库用户名 + :param password: 数据库密码 + :param db: 数据库名 + :return: 数据库连接 + """ + return connect(host=host, user=user, password=password, db=db) + + +def task_table_initialization( + database: connect, + table_name: str) -> bool: + """ + 初始化数据表，如果不存在则新建 + + :param database: 数据库连接 + :param table_name: 数据表名 + :return: True：初始化成功 | False：初始化失败 + """ + try: + with database.cursor() as database_cursor: + database_cursor.execute(f""" + create table if not exists {table_name}( + title text not null, + url text not null + )""") + database.commit() + database_cursor.close() + return True + except Error: + return False + + +def task_table_check( + database: connect, + table_name: str, + title: str, + url: str) -> bool: + """ + 检查表中是否已存在指定行。 + + :param database: 数据库连接 + :param table_name: 数据表名 + :param title: 标题 + :param url: 链接 + :return: True：已存在或出现错误 | False：不存在 + """ + try: + with database.cursor() as database_cursor: + result = database_cursor.execute(f"select * from {table_name} where title='{title}' and url='{url}'") + database_cursor.close() + if result > 0: + return True + else: + return False + except Error: + return True + + +def task_table_insert( + database: connect, + table_name: str, + title: str, + url: str): + """ + 向数据表中插入指定行。 + + :param database: 数据库连接 + :param table_name: 数据表名 + :param title: 标题 + :param url: 链接 + """ + try: + with database.cursor() as database_cursor: + database_cursor.execute(f"insert into {table_name} (title, url) VALUES ('{title}', '{url}')") + database.commit() + database_cursor.close() + except Error: + pass diff --git a/core/task.py b/core/task.py new file mode 100644 index 0000000..387b159 --- /dev/null +++ b/core/task.py @@ -0,0 +1,156 @@ +from email.mime.text import MIMEText +from email.utils import make_msgid, formataddr, parseaddr +from importlib import import_module +from json import dumps +from os import listdir +from smtplib import SMTPException, SMTP_SSL +from time import sleep +from urllib.parse import urljoin, urlparse + +from pymysql import connect +from pypinyin import slug + +from conf.push import push_setting +from conf.tasks import get_task_url, get_task_css +from core.parse import task_url_parse +from core.sql import task_table_initialization, task_table_check, task_table_insert + + +def _push_register() -> list: + """ + 注册推送方法。 + + :return: 推送方法列表 + """ + push_methods = [ + push_method_file[: push_method_file.rfind('.')] + for push_method_file in listdir('push') + if (push_method_file.endswith('.py')) and (push_method_file != 'base.py') + ] + for push_method_file_index in range(len(push_methods)): + push_method_module = import_module(f'push.{push_methods[push_method_file_index]}') + push_methods[push_method_file_index] = eval(f'push_method_module.{push_method_module.__dir__()[-1]}') + return push_methods + + +def _failure_process(failure_list: list): + try: + mail_server = SMTP_SSL(push_setting['smtp']['host']) + mail_server.connect( + push_setting['smtp']['host'], + push_setting['smtp']['port'] + ) + mail_server.login( + push_setting['smtp']['username'], + push_setting['smtp']['password'] + ) + + message = MIMEText( + dumps( + failure_list, + indent=4, + ensure_ascii=False + ), + 'plain', + 'utf-8' + ) + message['From'] = formataddr( + parseaddr(f"Inspection自动提醒 <{push_setting['smtp']['username']}>"), + 'utf-8' + ) + message['To'] = formataddr( + parseaddr(push_setting['admin']['email']), + 'utf-8' + ) + message['Subject'] = 'Inspection自动提醒：推送失败' + message['Message-ID'] = make_msgid() + + mail_server.sendmail( + push_setting['smtp']['username'], + push_setting['admin']['email'], + message.as_string() + ) + sleep(5) + mail_server.close() + return True + except SMTPException: + return False + + +class Task: + def __init__(self, + task: str, + subscribers: list, + sql_connect: connect): + """ + Inspection任务单元。 + + :param task: 任务名 + :param subscribers: 用户列表 + :param sql_connect: 数据库连接 + """ + self.task = task + self.url = get_task_url(task) + self.css_selector = get_task_css(task) + self.subscribers = subscribers + self.sql_connect = sql_connect + self.push_methods = _push_register() + + self.table_name = slug(task, separator='') + if task_table_initialization( + self.sql_connect, + self.table_name): + self._update() + + def _update(self): + retry_list = [] + failure_list = [] + + task_request = task_url_parse(self.url) + if task_request: + for item in task_request.select(self.css_selector): + write_title = item.text.strip() + write_url = urljoin( + f'{urlparse(self.url).scheme}://{urlparse(self.url).hostname}', + item.attrs['href'] + ) + + if not task_table_check( + self.sql_connect, + self.table_name, + write_title, + write_url): + for push_method in self.push_methods: + push_session = push_method( + self.task, + write_title, + write_url, + self.subscribers + ) + failed_push = push_session.get_failed_push() + if failed_push['subscribers']: + failed_push['class'] = push_method + retry_list.append(failed_push) + + task_table_insert( + self.sql_connect, + self.table_name, + write_title, + write_url + ) + + if retry_list: + for retry_push in retry_list: + retry_session = retry_push['class']( + retry_push['task'], + retry_push['title'], + retry_push['url'], + retry_push['subscribers'] + ) + failed_push = retry_session.get_failed_push() + if failed_push['subscribers']: + failed_push['class'] = retry_push['class'] + failure_list.append(failed_push) + + if failure_list: + _failure_process(failure_list) diff --git a/main.py b/main.py new file mode 100644 index 0000000..7a8839e --- /dev/null +++ b/main.py @@ -0,0 +1,23 @@ +from conf.sql import sql_setting +from conf.subscribers import subscribers +from conf.tasks import task_list +from core.sql import set_task_sql_connect +from core.task import Task + + +def main(): + sql_connect = set_task_sql_connect( + sql_setting['host'], + sql_setting['user'], + sql_setting['password'], + sql_setting['db'] + ) + + for task in task_list: + Task(task, subscribers, sql_connect) + + sql_connect.close() + + +if __name__ == '__main__': + main() diff --git a/push/base.py b/push/base.py new file mode 100644 index 0000000..b51f625 --- /dev/null +++ b/push/base.py @@ -0,0 +1,62 @@ +from abc import ABC, abstractmethod +from inspect import getfile +from os.path import basename + + +class PushBase(ABC): + def __init__(self, + task: str, + title: str, + url: str, + subscribers: list): + """ + 更新推送的抽象类，继承此类并实现_send_update方法即可注册为一个有效的推送方法。 + 注意：实现方法后请前往conf/subscribers.py文件中添加用户参数（必须包含enable参数）， + 前往conf/push.py添加该种推送方法所需（如有）的参数。 + + :param task: 任务名 + :param title: 消息标题 + :param url: 消息链接 + :param subscribers: 用户列表 + """ + push_name = basename(getfile(self.__class__))[: basename(getfile(self.__class__)).rfind('.')] + enabled_subscribers = [ + subscriber[push_name] + for subscriber in subscribers + if (subscriber[push_name]['enable']) and ((subscriber['range'] is True) or (task in subscriber['range'])) + ] + self._failed_push = { + 'task': task, + 'title': title, + 'url': url, + 'subscribers': [] + } + for subscriber in enabled_subscribers: + if not self._send_update(task, title, url, subscriber): + self._failed_push['subscribers'].append(subscriber) + + def get_failed_push(self): + """ + 当推送失败时，使用此方法获取未能成功送达的用户列表。 + + :return: 推送未能送达的用户列表及对应的消息内容 + """ + return self._failed_push + + @abstractmethod + def _send_update(self, + task: str, + title: str, + url: str, + subscriber: dict) -> bool: + """ + 推送的消息发送方式，推送类的最关键方法，必须在子类中实现才能使用。 + 请自行捕获异常，如果推送发送失败返回False，成功返回True。 + + :param task: 任务名 + :param title: 消息标题 + :param url: 消息链接 + :param subscriber: 用户 + :return: True：成功 | False：失败 + """ + pass diff --git a/push/mirai.py b/push/mirai.py new file mode 100644 index 0000000..d0b6ee7 --- /dev/null +++ b/push/mirai.py @@ -0,0 +1,86 @@ +from typing import Union + +from requests import post + +from conf.push import push_setting +from push.base import PushBase + + +class Mirai(PushBase): + def _send_update(self, + task: str, + title: str, + url: str, + subscriber: dict) -> bool: + def _mirai_http_auth() -> Union[str, bool]: + """ + Mirai HTTP接口身份验证。 + + :return: 如果完成身份验证，返回会话密钥，否则返回False + """ + try: + auth_string = { + 'verifyKey': push_setting['mirai']['verify_key'] + } + result = post(f'http://{push_setting["mirai"]["host"]}/verify', json=auth_string).json() + if int(result['code']) == 0: + bind_string = { + 'sessionKey': result['session'], + 'qq': push_setting['mirai']['sender'] + } + result = post(f'http://{push_setting["mirai"]["host"]}/bind', json=bind_string).json() + if int(result['code']) == 0: + return bind_string['sessionKey'] + else: + _mirai_http_release(bind_string['sessionKey']) + return False + else: + return False + except Exception: + return False + + def _mirai_http_release( + session_key: str) -> bool: + """ + 释放本次Mirai HTTP会话。 + + :param session_key: 待释放的会话密钥 + :return: True：成功 | False：失败 + """ + release_string = { + 'sessionKey': session_key, + 'qq': push_setting['mirai']['sender'] + } + try: + if int(post(f'http://{push_setting["mirai"]["host"]}/release', json=release_string).json()[ + 'code']) == 0: + return True + else: + return False + except Exception: + return False + + auth_result = _mirai_http_auth() + push_result = False + if auth_result: + headers = { + 'Content-Type': 'application/json' + } + message_string = { + 'sessionKey': auth_result, + 'target': subscriber['id'], + 'messageChain': [ + {'type': 'Plain', 'text': f'{task}发现更新：\n'}, + {'type': 'Plain', 'text': f'{title}\n'}, + {'type': 'Plain', 'text': url} + ] + } + request_format = 'sendGroupMessage' if subscriber['is_group'] else 'sendFriendMessage' + try: + if int(post(f'http://{push_setting["mirai"]["host"]}/{request_format}', json=message_string, + headers=headers).json()['code']) == 0: + push_result = True + except Exception: + pass + _mirai_http_release(auth_result) + return push_result diff --git a/push/smtp.py b/push/smtp.py new file mode 100644 index 0000000..32bbb2c --- /dev/null +++ b/push/smtp.py @@ -0,0 +1,52 @@ +from email.mime.text import MIMEText +from email.utils import make_msgid, parseaddr, formataddr +from smtplib import SMTPException, SMTP_SSL +from time import sleep + +from conf.push import push_setting +from push.base import PushBase + + +class SMTP(PushBase): + def _send_update(self, + task: str, + title: str, + url: str, + subscriber: dict) -> bool: + try: + mail_server = SMTP_SSL(push_setting['smtp']['host']) + mail_server.connect( + push_setting['smtp']['host'], + push_setting['smtp']['port'] + ) + mail_server.login( + push_setting['smtp']['username'], + push_setting['smtp']['password'] + ) + + message = MIMEText( + title + '\n' + url, + 'plain', + 'utf-8' + ) + message['From'] = formataddr( + parseaddr(f"Inspection自动提醒 <{push_setting['smtp']['username']}>"), + 'utf-8' + ) + message['To'] = formataddr( + parseaddr(subscriber['email']), + 'utf-8' + ) + message['Subject'] = f'Inspection自动提醒：{task}' + message['Message-ID'] = make_msgid() + + mail_server.sendmail( + push_setting['smtp']['username'], + subscriber['email'], + message.as_string() + ) + sleep(5) + mail_server.close() + return True + except SMTPException: + return False diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..804b518d8d4258a1a594467ecd0a44553308fa5b GIT binary patch literal 202 zcmXwzK?=e!5Jlfw@F=eI7z&CfXf{j>xcCPpF`YhvJT z?b)zo!O3}^r*S4%6Iox7ujuq=ZnET#clDQKd1j@CB0p*r?Zx~B(m9u Zw}ZP!RoMSGgRa0+6*}on^