From 2257b93593fbbe0967290a4b0369f4211a44379d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Thu, 5 Sep 2024 16:45:51 +0200 Subject: [PATCH 1/7] Adding Caddyfile code blocks to generate user-agents block list --- templates/Caddyfile.j2 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index c065baf..907fe07 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -5,6 +5,19 @@ {% for site in caddy_sites %} {{ site.domain }} { + + {% if site.useragent_blocklist is defined %} + @badbots { + {% for ua in site.useragent_blocklist %} + header User-Agent *{{ ua }}* + {% endfor %} + } + + handle @badbots { + abort + } + {% endif %} + {%- if site.allowlist is defined %} @allowlist { remote_ip {% for ip in site.allowlist %} {{ ip }}{% endfor %} From 819576cc67e8bf4ea8e6557a9c1ff4ddd00cbf13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Thu, 5 Sep 2024 17:29:00 +0200 Subject: [PATCH 2/7] Documenting new variable in README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b66a57c..e255c24 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ N/A If you only want to install Caddy, you don't need to set any variables. If you want to configure Caddy as a reverse proxy as well, you can provide an array of objects named `caddy_sites` with the following values: * `additional_forwarding_ports`: Allows to define a list with additional ports where Caddy should listen for this domain and forward to HTTPS. -* `allowlist`: An array if IP addresses in CIDR-notation which are allowed to access this site (Optional). All other visitors receive a 404 error. +* `allowlist`: An array of IP addresses in CIDR-notation which are allowed to access this site (Optional). All other visitors receive a 404 error. +* `useragent_blocklist`: An array of User-Agents which are blocked to access this site (Optional). * `certificate_file`: You can set this variable if you want to provide the certificate by yourself (Optional). The certificate needs permissions `0640`, with root as Owner and Caddy as Group. * `certificate_key`: You can set this variable if you want to provide the certificate by yourself (Optional). * `domain`: The domain caddy should listen to. From c1eed41469cd4f03c04c1676dec49c404611471b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Thu, 5 Sep 2024 17:33:28 +0200 Subject: [PATCH 3/7] Adding converge and Caddyfile.expected examples --- molecule/reverse-proxy/converge.yml | 2 ++ molecule/reverse-proxy/files/Caddyfile.expected | 8 ++++++++ templates/Caddyfile.j2 | 1 - 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/molecule/reverse-proxy/converge.yml b/molecule/reverse-proxy/converge.yml index 30ff6a8..372fa5f 100644 --- a/molecule/reverse-proxy/converge.yml +++ b/molecule/reverse-proxy/converge.yml @@ -20,6 +20,8 @@ code: 401 allowlist: - 8.8.8.8/32 + useragent_blocklist: + - amazonbot additional_forwarding_ports: - '8080' - '1337' diff --git a/molecule/reverse-proxy/files/Caddyfile.expected b/molecule/reverse-proxy/files/Caddyfile.expected index a22865b..b11715a 100644 --- a/molecule/reverse-proxy/files/Caddyfile.expected +++ b/molecule/reverse-proxy/files/Caddyfile.expected @@ -4,6 +4,14 @@ example.com { + @badbots { + header User-Agent *amazonbot* + } + + handle @badbots { + abort + } + @allowlist { remote_ip 8.8.8.8/32 } diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 907fe07..9377961 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -5,7 +5,6 @@ {% for site in caddy_sites %} {{ site.domain }} { - {% if site.useragent_blocklist is defined %} @badbots { {% for ua in site.useragent_blocklist %} From 0c1f83313ee5fcb0f257eff1a2c567147578008c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Mon, 9 Sep 2024 18:12:19 +0200 Subject: [PATCH 4/7] Changes according to Thomas comments --- templates/Caddyfile.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 9377961..2a3d766 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -5,17 +5,17 @@ {% for site in caddy_sites %} {{ site.domain }} { - {% if site.useragent_blocklist is defined %} + {%- if site.useragent_blocklist is defined %} @badbots { - {% for ua in site.useragent_blocklist %} + {%- for ua in site.useragent_blocklist %} header User-Agent *{{ ua }}* - {% endfor %} + {%- endfor %} } handle @badbots { abort } - {% endif %} + {%- endif %} {%- if site.allowlist is defined %} @allowlist { From a8636b78223a6b78a10373b50db50dbeb5063d1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Fri, 13 Sep 2024 11:47:03 +0200 Subject: [PATCH 5/7] Replying 403 code and basic access forbidden content, instead of closing TCP connection --- templates/Caddyfile.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index 2a3d766..ba36b12 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -13,8 +13,11 @@ } handle @badbots { - abort + respond 403 { + body "Access forbidden." + } } + {%- endif %} {%- if site.allowlist is defined %} From 93bc0532aee7d0ec13269bc6ed6d8c4453de0931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Fri, 13 Sep 2024 11:48:58 +0200 Subject: [PATCH 6/7] Adapting Caddyfile.excepted to new 403 response --- molecule/reverse-proxy/files/Caddyfile.expected | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/molecule/reverse-proxy/files/Caddyfile.expected b/molecule/reverse-proxy/files/Caddyfile.expected index b11715a..ffda6d7 100644 --- a/molecule/reverse-proxy/files/Caddyfile.expected +++ b/molecule/reverse-proxy/files/Caddyfile.expected @@ -9,7 +9,9 @@ example.com { } handle @badbots { - abort + respond 403 { + body "Access forbidden." + } } @allowlist { From b5427b39b468a13addb4639d4c55ab28ec7639de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robin=20Fran=C3=A7ois?= Date: Wed, 25 Sep 2024 14:51:19 +0200 Subject: [PATCH 7/7] Removing wildcard in the ansible role. Wildcards will have to be added when defining the variables --- README.md | 2 +- molecule/reverse-proxy/converge.yml | 2 +- templates/Caddyfile.j2 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e255c24..002e2f3 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ If you only want to install Caddy, you don't need to set any variables. If you w * `additional_forwarding_ports`: Allows to define a list with additional ports where Caddy should listen for this domain and forward to HTTPS. * `allowlist`: An array of IP addresses in CIDR-notation which are allowed to access this site (Optional). All other visitors receive a 404 error. -* `useragent_blocklist`: An array of User-Agents which are blocked to access this site (Optional). +* `useragent_blocklist`: An array of User-Agents which are blocked to access this site (Optional), wildcard characters (*) need to be used for broader matching. * `certificate_file`: You can set this variable if you want to provide the certificate by yourself (Optional). The certificate needs permissions `0640`, with root as Owner and Caddy as Group. * `certificate_key`: You can set this variable if you want to provide the certificate by yourself (Optional). * `domain`: The domain caddy should listen to. diff --git a/molecule/reverse-proxy/converge.yml b/molecule/reverse-proxy/converge.yml index 372fa5f..cb70736 100644 --- a/molecule/reverse-proxy/converge.yml +++ b/molecule/reverse-proxy/converge.yml @@ -21,7 +21,7 @@ allowlist: - 8.8.8.8/32 useragent_blocklist: - - amazonbot + - "*amazonbot*" additional_forwarding_ports: - '8080' - '1337' diff --git a/templates/Caddyfile.j2 b/templates/Caddyfile.j2 index ba36b12..351b6b1 100644 --- a/templates/Caddyfile.j2 +++ b/templates/Caddyfile.j2 @@ -8,7 +8,7 @@ {%- if site.useragent_blocklist is defined %} @badbots { {%- for ua in site.useragent_blocklist %} - header User-Agent *{{ ua }}* + header User-Agent {{ ua }} {%- endfor %} }