From 2cf3d5d911573a1175038cae1cc83778e0c722ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Thu, 5 Sep 2024 16:45:51 +0200 Subject: [PATCH 1/7] Adding Caddyfile code blocks to generate user-agents block list --- templates/Caddyfile.j2 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index c065baf..907fe07 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -5,6 +5,19 @@ {% for site in caddy_sites %} {{ site.domain }} { + + {% if site.useragent_blocklist is defined %} + @badbots { + {% for ua in site.useragent_blocklist %} + header User-Agent *{{ ua }}* + {% endfor %} + } + + handle @badbots { + abort + } + {% endif %} + {%- if site.allowlist is defined %} @allowlist { remote_ip {% for ip in site.allowlist %} {{ ip }}{% endfor %} From db40194c117375d20808a52506cd338cd17e8b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Thu, 5 Sep 2024 17:29:00 +0200 Subject: [PATCH 2/7] Documenting new variable in README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b66a57c..e255c24 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ N/A If you only want to install Caddy, you don't need to set any variables. If you want to configure Caddy as a reverse proxy as well, you can provide an array of objects named `caddy_sites` with the following values: * `additional_forwarding_ports`: Allows to define a list with additional ports where Caddy should listen for this domain and forward to HTTPS. -* `allowlist`: An array if IP addresses in CIDR-notation which are allowed to access this site (Optional). All other visitors receive a 404 error. +* `allowlist`: An array of IP addresses in CIDR-notation which are allowed to access this site (Optional). All other visitors receive a 404 error. +* `useragent_blocklist`: An array of User-Agents which are blocked to access this site (Optional). * `certificate_file`: You can set this variable if you want to provide the certificate by yourself (Optional). The certificate needs permissions `0640`, with root as Owner and Caddy as Group. * `certificate_key`: You can set this variable if you want to provide the certificate by yourself (Optional). * `domain`: The domain caddy should listen to. From 32409cbc6bef439947fd192395e9fe0825def593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Thu, 5 Sep 2024 17:33:28 +0200 Subject: [PATCH 3/7] Adding converge and Caddyfile.expected examples --- molecule/reverse-proxy/converge.yml | 2 ++ molecule/reverse-proxy/files/Caddyfile.expected | 8 ++++++++ templates/Caddyfile.j2 | 1 - 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/molecule/reverse-proxy/converge.yml b/molecule/reverse-proxy/converge.yml index 30ff6a8..372fa5f 100644 --- a/molecule/reverse-proxy/converge.yml +++ b/molecule/reverse-proxy/converge.yml @@ -20,6 +20,8 @@ code: 401 allowlist: - 8.8.8.8/32 + useragent_blocklist: + - amazonbot additional_forwarding_ports: - '8080' - '1337' diff --git a/molecule/reverse-proxy/files/Caddyfile.expected b/molecule/reverse-proxy/files/Caddyfile.expected index a22865b..b11715a 100644 --- a/molecule/reverse-proxy/files/Caddyfile.expected +++ b/molecule/reverse-proxy/files/Caddyfile.expected @@ -4,6 +4,14 @@ example.com { + @badbots { + header User-Agent *amazonbot* + } + + handle @badbots { + abort + } + @allowlist { remote_ip 8.8.8.8/32 } diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 907fe07..9377961 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -5,7 +5,6 @@ {% for site in caddy_sites %} {{ site.domain }} { - {% if site.useragent_blocklist is defined %} @badbots { {% for ua in site.useragent_blocklist %} From eac78531011aa8ee15d1e00a799cb6f4d3c1f1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Mon, 9 Sep 2024 18:12:19 +0200 Subject: [PATCH 4/7] Changes according to Thomas comments --- templates/Caddyfile.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 9377961..2a3d766 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -5,17 +5,17 @@ {% for site in caddy_sites %} {{ site.domain }} { - {% if site.useragent_blocklist is defined %} + {%- if site.useragent_blocklist is defined %} @badbots { - {% for ua in site.useragent_blocklist %} + {%- for ua in site.useragent_blocklist %} header User-Agent *{{ ua }}* - {% endfor %} + {%- endfor %} } handle @badbots { abort } - {% endif %} + {%- endif %} {%- if site.allowlist is defined %} @allowlist { From a06df43ca2be38a62ee38c4b198056b5013a72b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Fri, 13 Sep 2024 11:47:03 +0200 Subject: [PATCH 5/7] Replying 403 code and basic access forbidden content, instead of closing TCP connection --- templates/Caddyfile.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 2a3d766..ba36b12 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -13,8 +13,11 @@ } handle @badbots { - abort + respond 403 { + body "Access forbidden." + } } + {%- endif %} {%- if site.allowlist is defined %} From 1b14fb78ee13f1368f4ec7de69ce240788ec34a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Fri, 13 Sep 2024 11:48:58 +0200 Subject: [PATCH 6/7] Adapting Caddyfile.excepted to new 403 response --- molecule/reverse-proxy/files/Caddyfile.expected | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/molecule/reverse-proxy/files/Caddyfile.expected b/molecule/reverse-proxy/files/Caddyfile.expected index b11715a..ffda6d7 100644 --- a/molecule/reverse-proxy/files/Caddyfile.expected +++ b/molecule/reverse-proxy/files/Caddyfile.expected @@ -9,7 +9,9 @@ example.com { } handle @badbots { - abort + respond 403 { + body "Access forbidden." + } } @allowlist { From fba99e396745af71d1ad4986c02f060c74a19f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Wed, 25 Sep 2024 14:51:19 +0200 Subject: [PATCH 7/7] Removing wildcard in the ansible role. Wildcards will have to be added when defining the variables --- README.md | 2 +- molecule/reverse-proxy/converge.yml | 2 +- templates/Caddyfile.j2 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e255c24..002e2f3 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ If you only want to install Caddy, you don't need to set any variables. If you w * `additional_forwarding_ports`: Allows to define a list with additional ports where Caddy should listen for this domain and forward to HTTPS. * `allowlist`: An array of IP addresses in CIDR-notation which are allowed to access this site (Optional). All other visitors receive a 404 error. -* `useragent_blocklist`: An array of User-Agents which are blocked to access this site (Optional). +* `useragent_blocklist`: An array of User-Agents which are blocked to access this site (Optional), wildcard characters (*) need to be used for broader matching. * `certificate_file`: You can set this variable if you want to provide the certificate by yourself (Optional). The certificate needs permissions `0640`, with root as Owner and Caddy as Group. * `certificate_key`: You can set this variable if you want to provide the certificate by yourself (Optional). * `domain`: The domain caddy should listen to. diff --git a/molecule/reverse-proxy/converge.yml b/molecule/reverse-proxy/converge.yml index 372fa5f..cb70736 100644 --- a/molecule/reverse-proxy/converge.yml +++ b/molecule/reverse-proxy/converge.yml @@ -21,7 +21,7 @@ allowlist: - 8.8.8.8/32 useragent_blocklist: - - amazonbot + - "*amazonbot*" additional_forwarding_ports: - '8080' - '1337' diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index ba36b12..351b6b1 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -8,7 +8,7 @@ {%- if site.useragent_blocklist is defined %} @badbots { {%- for ua in site.useragent_blocklist %} - header User-Agent *{{ ua }}* + header User-Agent {{ ua }} {%- endfor %} }