Merge pull request #5638 from LMFDB/master

master -> web
LMFDB · Aug 10, 2023 · 4d358da · 4d358da
2 parents dec74c5 + 3e27d9f
commit 4d358da
Showing 1 changed file with 13 additions and 0 deletions.
diff --git a/lmfdb/app.py b/lmfdb/app.py
@@ -297,6 +297,16 @@ def bad_bots_list():
         for elt in [
             "The Knowledge AI",
             "Wolfram",
+            "petalbot",
+        ]
+    ]
+
+
+@cached_function
+def very_bad_bots_list():
+    return [
+        elt.lower()
+        for elt in [
             "Amazonbot",
         ]
     ]
@@ -305,6 +315,9 @@ def bad_bots_list():
 @app.before_request
 def badbot():
     ua = request.user_agent.string.lower()
+    for elt in very_bad_bots_list():
+        if elt in ua:
+            return render_template("404.html", title='Too many requests'), 429
     for elt in bad_bots_list():
         if elt in ua:
             time.sleep(10)