From 8d75174805b389d7921edc06eda7195e69dcb27f Mon Sep 17 00:00:00 2001 From: cdhigh Date: Mon, 15 Apr 2024 21:13:56 -0300 Subject: [PATCH] 3.0.0e 1. add docker image kindleear/mailfix 2. add caddyfile --- application/templates/base.html | 6 +-- docker/Caddyfile | 8 ++++ docker/Dockerfile | 30 +++++---------- docker/default.conf | 65 ++++++++++----------------------- docker/docker-compose-nginx.yml | 54 +++++++++++++++++++++++++++ docker/docker-compose.yml | 46 +++++++++++++---------- docker/postfix/Dockerfile | 12 ++++++ docker/postfix/install.sh | 53 +++++++++++++++++++++++++++ docker/postfix/readme.md | 9 +++++ docs/Chinese/changelog.md | 2 +- docs/Chinese/deployment.md | 59 +++++++++++++++++++++--------- docs/English/changelog_en.md | 2 +- docs/English/deployment.md | 55 ++++++++++++++++++++-------- docs/English/faq.md | 2 +- tools/update_req.py | 9 ++++- 15 files changed, 284 insertions(+), 128 deletions(-) create mode 100644 docker/Caddyfile create mode 100644 docker/docker-compose-nginx.yml create mode 100644 docker/postfix/Dockerfile create mode 100644 docker/postfix/install.sh create mode 100644 docker/postfix/readme.md diff --git a/application/templates/base.html b/application/templates/base.html index 9263d625..a85f9b67 100644 --- a/application/templates/base.html +++ b/application/templates/base.html @@ -172,9 +172,9 @@ {% endblock -%} - - + +'); + {% block jsfiles %}{% endblock -%} diff --git a/docker/Caddyfile b/docker/Caddyfile new file mode 100644 index 00000000..a9bded5b --- /dev/null +++ b/docker/Caddyfile @@ -0,0 +1,8 @@ +{$DOMAIN} { + log { + output stdout + format console + level ERROR + } + reverse_proxy kindleear:8000 +} diff --git a/docker/Dockerfile b/docker/Dockerfile index f65656e1..94179ccc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,32 +1,20 @@ -#step 1 -#FROM python:3.10.14-slim AS req_builder -FROM python:3.9.19-alpine AS req_builder -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 - -WORKDIR /usr/site/ -COPY ./config.py . -COPY ./tools/update_req.py . -RUN python update_req.py docker - -#step 2 #FROM python:3.10.14-slim FROM python:3.9.19-alpine ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 USER root -RUN mkdir -p /usr/site /data -WORKDIR /usr/site -RUN pip install --upgrade pip -COPY --from=req_builder /usr/site/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN mkdir -p /usr/kindleear /data +WORKDIR /usr/kindleear +COPY ./config.py ./tools/update_req.py ./docker/gunicorn.conf.py ./main.py ./ + +#RUN python update_req.py docker-all +RUN python update_req.py docker && \ + pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt -COPY ./docker/gunicorn.conf.py . -COPY ./main.py . -COPY --from=req_builder /usr/site/config.py . COPY ./application/ ./application/ EXPOSE 8000 -CMD ["/usr/local/bin/gunicorn", "-c", "/usr/site/gunicorn.conf.py", "main:app"] +CMD ["/usr/local/bin/gunicorn", "-c", "/usr/kindleear/gunicorn.conf.py", "main:app"] diff --git a/docker/default.conf b/docker/default.conf index 92fe5cf2..69f6c85c 100644 --- a/docker/default.conf +++ b/docker/default.conf @@ -1,48 +1,23 @@ server { - listen 80 default_server; - listen [::]:80 default_server; - charset utf-8; - client_max_body_size 32M; + listen 80 default_server; + listen [::]:80 default_server; + charset utf-8; + client_max_body_size 32M; + server_name localhost; - # SSL configuration - # - # listen 443 ssl default_server; - # listen [::]:443 ssl default_server; - # - # Note: You should disable gzip for SSL traffic. - # See: https://bugs.debian.org/773332 - # - # Read up on ssl_ciphers to ensure a secure configuration. - # See: https://bugs.debian.org/765782 - # - # Self signed certs generated by the ssl-cert package - # Don't use them in a production server! - # - # include snippets/snakeoil.conf; - - root /var/www/html; - - # Add index.php to the list if you are using PHP - index index.html index.htm index.nginx-debian.html; - - server_name localhost; - - location /static/ { - proxy_pass http://kindleear:8000/static/; - } - location /images/ { - proxy_pass http://kindleear:8000/images/; - } - location = /favicon.ico { - proxy_pass http://kindleear:8000/static/favicon.ico; - } - location = /robots.txt { - proxy_pass http://kindleear:8000/static/robots.txt; - } - location / { - proxy_pass http://kindleear:8000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } + # uncomment this section if https is need + listen 443 ssl default_server; + listen [::]:443 ssl default_server; + ssl_certificate /etc/nginx/ssl/fullchain.pem; + ssl_certificate_key /etc/nginx/ssl/privkey.pem; + if ($scheme = http) { + return 301 https://$http_host$request_uri; + } + + location / { + proxy_pass http://kindleear:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } } diff --git a/docker/docker-compose-nginx.yml b/docker/docker-compose-nginx.yml new file mode 100644 index 00000000..6adae4f2 --- /dev/null +++ b/docker/docker-compose-nginx.yml @@ -0,0 +1,54 @@ + +services: + kindleear: + container_name: kindleear + image: kindleear/kindleear + restart: always + volumes: + - ./data/:/data/ + expose: + - "8000" + networks: + - web_network + environment: + APP_ID: kindleear + APP_DOMAIN: http://example.com + LOG_LEVEL: warning + + nginx: + container_name: nginx + image: nginx:stable-alpine3.17-slim + restart: always + ports: + - "80:80" + - "443:443" + volumes: + - ./default.conf:/etc/nginx/conf.d/default.conf:ro + - ./data/:/var/log/nginx/ + # uncomment this two lines if https is need + #- ./data/fullchain.pem:/etc/nginx/ssl/fullchain.pem:ro + #- ./data/privkey.pem:/etc/nginx/ssl/privkey.pem:ro + depends_on: + - kindleear + networks: + - web_network + + mailfix: + container_name: mailfix + image: kindleear/mailfix + restart: unless-stopped + ports: + - "25:25" + depends_on: + - kindleear + - nginx + environment: + #change DOMAIN to your email domain, without http and https prefix + DOMAIN: example.com + URL: http://kindleear:8000/mail + networks: + - web_network + +networks: + web_network: + driver: bridge diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e748e05e..1001adb1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,37 +12,43 @@ services: - web_network environment: APP_ID: kindleear - APP_DOMAIN: https://kindleear.appspot.com + #DOMAIN with http or https prefix + APP_DOMAIN: http://example.com LOG_LEVEL: warning - nginx: - container_name: nginx - image: nginx:stable-alpine3.17-slim + caddy: + container_name: caddy + image: caddy:alpine restart: always ports: - "80:80" + - "443:443" + environment: + #DOMAIN without http and https prefix + DOMAIN: example.com volumes: - - ./default.conf:/etc/nginx/conf.d/default.conf:ro + - ./Caddyfile:/etc/caddy/Caddyfile + - ./caddy/:/data/caddy/ depends_on: - kindleear networks: - web_network - # mailglove: - # container_name: mailglove - # image: thingless/mailglove - # restart: unless-stopped - # ports: - # - "25:25" - # depends_on: - # - kindleear - # - nginx - # environment: - # #change DOMAIN to your email domain - # DOMAIN: local - # URL: http://kindleear:8000/mailglove - # networks: - # - web_network + mailfix: + container_name: mailfix + image: kindleear/mailfix + restart: unless-stopped + ports: + - "25:25" + depends_on: + - kindleear + - nginx + environment: + #change DOMAIN to your email domain, without http and https prefix + DOMAIN: example.com + URL: http://kindleear:8000/mail + networks: + - web_network networks: web_network: diff --git a/docker/postfix/Dockerfile b/docker/postfix/Dockerfile new file mode 100644 index 00000000..d07e483d --- /dev/null +++ b/docker/postfix/Dockerfile @@ -0,0 +1,12 @@ +# Inspired by +From alpine:3.19.1 + +ADD install.sh /etc/postfix/install.sh + +RUN apk add --no-cache curl bash postfix && \ + newaliases && \ + chmod 755 /etc/postfix/install.sh + +EXPOSE 25 + +CMD ["/etc/postfix/install.sh"] diff --git a/docker/postfix/install.sh b/docker/postfix/install.sh new file mode 100644 index 00000000..e78c82da --- /dev/null +++ b/docker/postfix/install.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +#steal from https://github.com/thingless/mailglove +postconf -e myhostname=${DOMAIN} + +# Add the myhook hook to the end of master.cf +if ! grep -qF 'myhook unix - n n - - pipe' /etc/postfix/master.cf; then +tee -a /etc/postfix/master.cf < ，功能完全一样。 +功能是拦截postfix接收到的所有邮件，然后调用通过环境变量传入的URL的webhook。 + +# 为什么有了mailglove还需要制作mailfix？ +mailglove太大了，解压前124MB，解压后338M，为了这么一个简单的功能消耗那么大的空间实在不值得。 +所以我就使用alpine代替ubuntu，使用sh代替nodejs。 +除了alpine镜像和postfix，实际上只有一个sh文件，镜像解压后只有26.8M。 + diff --git a/docs/Chinese/changelog.md b/docs/Chinese/changelog.md index 9f676ada..95452fa6 100644 --- a/docs/Chinese/changelog.md +++ b/docs/Chinese/changelog.md @@ -1,5 +1,5 @@ --- -sort: 5 +sort: 6 --- # Changelog diff --git a/docs/Chinese/deployment.md b/docs/Chinese/deployment.md index 6f79ce3f..297310e8 100644 --- a/docs/Chinese/deployment.md +++ b/docs/Chinese/deployment.md @@ -82,51 +82,68 @@ wget -O - https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/ubunt 2. 安装完Docker后，执行一条命令就可以让服务运行起来（yourdomain修改为你自己的值）。 命令执行后就使用浏览器 http://ip 确认服务是否正常运行。 因为使用了 restart 参数，所以系统重启后会自动重启此服务。 -注：这条命令采用默认配置： -* sqlite数据库 -* apscheduler，内存队列 -* 数据库文件和log文件保存到同一目录 /data -如果你需要其他配置组合，根据config.py的变量名传入对应的环境变量即可(-e 参数)。 - ```bash mkdir data #for database and logs, you can use any folder (change ./data to your folder) sudo docker run -d -p 80:8000 -v ./data:/data --restart always -e APP_DOMAIN=yourdomain kindleear/kindleear ``` +注：默认镜像的配置： +* sqlite数据库 +* apscheduler，内存队列 +* 数据库文件和log文件保存到同一目录 /data +如果你需要使用其他数据库或任务队列，可以使用Dockerfile直接构建镜像。 如果连不上，请确认80端口是否已经开放，不同的平台开放80端口的方法不一样，可能为iptables或ufw。 比如： ```bash sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 80 -j ACCEPT -sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 443 -j ACCEPT +sudo iptables -I INPUT 7 -m state --state NEW -p tcp --dport 443 -j ACCEPT sudo netfilter-persistent save ``` -3. 上面的命令直接使用gunicorn为web服务器，尽管对于我们的应用来说，绰绰有余，而且还能省点资源。 -但是你感觉有些不专业，希望使用更强大的nginx，并且也想启用邮件中转功能（需要开放25端口并且申请域名）： +如果需要https支持，可以申请一个SSL证书，然后通过环境变量传递给gunicorn，比如到 let's encrypt 申请一个免费证书，然后将 fullchain.pem/privkey.pem拷贝到data目录，再执行此命令 +```bash +sudo docker run -d -p 80:8000 -p 443:8000 -v ./data:/data --restart always -e APP_DOMAIN=https://kindleear.line.pm -e GUNI_CERT=/data/fullchain.pem -e GUNI_KEY=/data/privkey.pem kindleear/kindleear +``` +3. 如果需要使用https，更推荐的是使用caddy做为web服务器，可以自动申请和续期ssl证书（一定要正确填写DOMAIN）： ```bash mkdir data #for database and logs wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/docker-compose.yml -wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/default.conf +wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/Caddyfile -#Change the environ variables APP_ID/APP_DOMAIN -#If the email feature is needed, uncomment the section mailglove and change the DOMAIN. +#important!!! Change the environ variables APP_DOMAIN/DOMAIN vim ./docker-compose.yml sudo docker compose up -d ``` -4. 需要查询日志文件 + +4. 如果更喜欢nginx： ```bash -tail -n 100 ./data/gunicorn.error.log -tail -n 100 ./data/gunicorn.access.log +mkdir data #for database and logs +wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/docker-compose-nginx.yml +wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/default.conf + +#Change the environ variables APP_DOMAIN/DOMAIN +vim ./docker-compose-nginx.yml + +sudo docker compose -f docker-compose-nginx.yml up -d +``` + +使用nginx时如果需要https，预先将ssl证书 fullchain.pem/privkey.pem 拷贝到data目录，取消default.conf/docker-compose-nginx.yml里面对应的注释即可。 + + +5. 需要查询日志文件 +```bash +tail -n 50 ./data/gunicorn.error.log +tail -n 50 ./data/gunicorn.access.log ``` ## Oracle cloud (VPS) -这是手动在一个VPS上部署的步骤，比较复杂，一般不建议，如果没有特殊要求，推荐使用docker镜像。 +这是手动在一个 [Oracle VPS](https://cloud.oracle.com/) 上部署的步骤，比较复杂，一般不建议，如果没有特殊要求，推荐使用docker镜像。 1. config.py关键参数样例 ```python DATABASE_URL = "sqlite:////home/ubuntu/site/kindleear/database.db" @@ -138,7 +155,13 @@ DOWNLOAD_THREAD_NUM = "3" 2. 创建一个计算实例，选择的配置建议"符合始终免费条件"，镜像选择自己熟悉的，我选择的是ubuntu minimal。 记得下载和保存私钥。 -创建完成后在"实例信息"点击"子网"链接，在"安全列表"中修改或创建入站规则，将TCP的端口删除，ICMP的类型和代码删除，然后测试ping对应的IP，能ping通说明实例配置完成。 +创建完成后在"实例信息"点击"子网"链接，在"安全列表"中修改或创建入站规则，将TCP的端口删除，ICMP的类型和代码删除， +只保留一个入站规则： +源类型：CIDR +源CIDR：0.0.0.0/0 +IP协议：所有协议 + +然后测试ping对应的IP，能ping通说明实例配置完成。 3. 使用自己喜欢的SSH工具远程连接对应IP。 3.1 如果使用puTTY，需要先使用puttyGen将key格式的私钥转换为ppk格式。 @@ -186,7 +209,7 @@ python3 ./main.py db create #create database tables #open port 80/443 sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 80 -j ACCEPT -sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 443 -j ACCEPT +sudo iptables -I INPUT 7 -m state --state NEW -p tcp --dport 443 -j ACCEPT sudo netfilter-persistent save mkdir -p /var/log/gunicorn/ diff --git a/docs/English/changelog_en.md b/docs/English/changelog_en.md index 9436d23c..b9e7709b 100644 --- a/docs/English/changelog_en.md +++ b/docs/English/changelog_en.md @@ -1,5 +1,5 @@ --- -sort: 5 +sort: 6 --- # Changelog diff --git a/docs/English/deployment.md b/docs/English/deployment.md index 63357beb..05e02334 100644 --- a/docs/English/deployment.md +++ b/docs/English/deployment.md @@ -86,50 +86,67 @@ wget -O - https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/ubunt 2. Execute a command to start the service (replace `yourid/yourdomain` with your own values). Confirm if the service is running properly by visiting http://ip in a browser. The service will automatically restart after system reboot due to the `restart` parameter. +```bash +mkdir data #for database and logs, you can use any folder (change ./data to your folder) +sudo docker run -d -p 80:8000 -v ./data:/data --restart always -e APP_DOMAIN=yourdomain kindleear/kindleear +``` Note: This command uses the default configuration: * SQLite database * APScheduler, memory job store * Database and log files are saved to the same directory `/data` -If you need a different configuration combination, simply pass the corresponding environment variables based on the variable names in `config.py` (-e parameter). +If you need to use other databases or task queues, you can build the custom image using Dockerfile. -```bash -mkdir data #for database and logs, you can use any folder (change ./data to your folder) -sudo docker run -d -p 80:8000 -v ./data:/data --restart always -e APP_DOMAIN=yourdomain kindleear/kindleear -``` If unable to connect, ensure port 80 is open. Methods to open port 80 vary across platforms, such as iptables or ufw. For example: ```bash sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 80 -j ACCEPT -sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 443 -j ACCEPT +sudo iptables -I INPUT 7 -m state --state NEW -p tcp --dport 443 -j ACCEPT sudo netfilter-persistent save ``` -3. The above command directly uses Gunicorn as the web server, which is more than sufficient for our application and saves resources. -However, if you feel it's unprofessional and want to use the more powerful Nginx, and also enable email forwarding (requires opening port 25 and registering a domain): +If HTTPS support is needed, you can apply for an SSL certificate, then pass it to Gunicorn through environment variables. For example, you can apply for a free certificate from "Let's Encrypt", then copy `fullchain.pem/privkey.pem` to the data directory, and execute this command. +```bash +sudo docker run -d -p 80:8000 -p 443:8000 -v ./data:/data --restart always -e APP_DOMAIN=yourdomain -e GUNI_CERT=/data/fullchain.pem -e GUNI_KEY=/data/privkey.pem kindleear/kindleear +``` + +3. If HTTPS is needed, it is more recommended to use Caddy as the web server, which can automatically request and renew SSL certificates. ```bash mkdir data #for database and logs wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/docker-compose.yml -wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/default.conf +wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/Caddyfile -# Change the environment variables APP_ID/APP_DOMAIN -# If the email feature is needed, uncomment the section mailglove and change the DOMAIN. +#important!!! Change the environ variables APP_DOMAIN/DOMAIN vim ./docker-compose.yml sudo docker compose up -d ``` -4. To check log files: +4. If you perfer nginx. +```bash +mkdir data #for database and logs +wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/docker-compose-nginx.yml +wget https://raw.githubusercontent.com/cdhigh/KindleEar/master/docker/default.conf + +# Change the environment variables APP_DOMAIN/DOMAIN +vim ./docker-compose-nginx.yml + +sudo docker compose -f docker-compose-nignx.yml up -d +``` + +If HTTPS for nginx is needed, copy the SSL certificate fullchain.pem/privkey.pem to the data directory, and uncomment the corresponding lines in default.conf/docker-compose-nginx.yml. + +5. To check log files: ```bash -tail -n 100 ./data/gunicorn.error.log -tail -n 100 ./data/gunicorn.access.log +tail -n 50 ./data/gunicorn.error.log +tail -n 50 ./data/gunicorn.access.log ``` ## Oracle cloud (VPS) -These are manual deployment steps on a VPS, which can be quite complex. Generally, it's not recommended. +These are manual deployment steps on [Oracle VPS](https://cloud.oracle.com/) , which can be quite complex. Generally, it's not recommended. If there are no specific requirements, it's advisable to use Docker images instead. 1. config.py Key Parameter Example @@ -144,6 +161,11 @@ DOWNLOAD_THREAD_NUM = "3" 2. Create a compute instance, with the recommended configuration being "Always Free". Choose an image that you are familiar with, I selected Ubuntu minimal. Remember to download and save the private ssh key. Once created, click on the "Subnet" link on "Instance Details" page then modify or create inbound rules in the "Security Lists" by removing TCP ports and ICMP types and codes. +Keep only one Ingress Rule: +Source Type: CIDR +Source CIDR: 0.0.0.0/0 +IP Protocol: All protocols + Test ping the corresponding IP, if successful, it indicates that the instance configuration is complete. 3. Connect remotely to the instance using your preferred SSH tool. @@ -190,8 +212,9 @@ source ./venv/bin/activate #activate virtual environ pip install -r requirements.txt #install dependencies python3 ./main.py db create #create database tables -#open port 80 +#open port 80/443 sudo iptables -I INPUT 6 -m state --state NEW -p tcp --dport 80 -j ACCEPT +sudo iptables -I INPUT 7 -m state --state NEW -p tcp --dport 443 -j ACCEPT sudo netfilter-persistent save #modify nginx configuration diff --git a/docs/English/faq.md b/docs/English/faq.md index 1c7cdc1f..5aefad31 100644 --- a/docs/English/faq.md +++ b/docs/English/faq.md @@ -1,5 +1,5 @@ --- -sort: 4 +sort: 5 --- # FAQ diff --git a/tools/update_req.py b/tools/update_req.py index 8c9676d2..f2d52b8f 100644 --- a/tools/update_req.py +++ b/tools/update_req.py @@ -176,6 +176,11 @@ def gaeify_config_py(cfgFile): f.write('\n'.join(ret)) print(f'Finished update {cfgFile}') +#command arugments: +# update_req docker : prepare for build docker image +# update_req docker-all : prepare for build docker image, install all libs +# update_req gae : prepare for deploying in gae +# update_req : do not modify config.py, only update requirements.txt if __name__ == '__main__': thisDir = os.path.abspath(os.path.dirname(__file__)) cfgFile = os.path.normpath(os.path.join(thisDir, '..', 'config.py')) @@ -186,10 +191,10 @@ def gaeify_config_py(cfgFile): dockerize = False gaeify = False - if len(sys.argv) == 2 and sys.argv[1] == 'docker': + if len(sys.argv) == 2 and sys.argv[1] in ('docker', 'docker-all'): print('\nGenerating config.py and requirements.txt for Docker deployment.\n') dockerize_config_py(cfgFile) - dockerize = True + dockerize = (sys.argv[1] == 'docker-all') elif len(sys.argv) == 2 and sys.argv[1] == 'gae': print('\nGenerating config.py and requirements.txt for GAE deployment.\n') gaeify_config_py(cfgFile)