diff --git a/lmfdb/app.py b/lmfdb/app.py index bc0d650d90..0a02d9b261 100644 --- a/lmfdb/app.py +++ b/lmfdb/app.py @@ -297,6 +297,16 @@ def bad_bots_list(): for elt in [ "The Knowledge AI", "Wolfram", + "petalbot", + ] + ] + + +@cached_function +def very_bad_bots_list(): + return [ + elt.lower() + for elt in [ "Amazonbot", ] ] @@ -305,6 +315,9 @@ def bad_bots_list(): @app.before_request def badbot(): ua = request.user_agent.string.lower() + for elt in very_bad_bots_list(): + if elt in ua: + return render_template("404.html", title='Too many requests'), 429 for elt in bad_bots_list(): if elt in ua: time.sleep(10)