Merge pull request #480 from liberapay/postgres-down

Handle Postgres downtime better

Makefile

@@ -45,7 +45,8 @@ data: env
 db-migrations: sql/migrations.sql
 	PYTHONPATH=. $(with_local_env) $(env_py) liberapay/models/__init__.py
 
-run: env db-migrations
+run: env
+	@$(MAKE) --no-print-directory db-migrations || true
 	PATH=$(env_bin):$$PATH $(with_local_env) $(env_py) app.py
 
 py: env

defaults.env

@@ -6,6 +6,9 @@ PYTHONDONTWRITEBYTECODE=true
 CANONICAL_HOST=localhost:8339
 CANONICAL_SCHEME=http
 
+COMPRESS_ASSETS=no
+CSP_EXTRA=
+
 OAUTHLIB_INSECURE_TRANSPORT=1
 OAUTHLIB_RELAX_TOKEN_SCOPE=1
 

liberapay/exceptions.py

@@ -42,6 +42,16 @@ def lazy_body(self, _):
         return _("You need to log in")
 
 
+class NeedDatabase(LazyResponse):
+
+    def __init__(self):
+        Response.__init__(self, 503, '')
+        self.html_template = 'templates/no-db.html'
+
+    def lazy_body(self, _):
+        return _("We're unable to process your request right now, sorry.")
+
+
 class LazyResponseXXX(LazyResponse):
 
     def __init__(self, *args, **kw):

liberapay/main.py

@@ -1,6 +1,9 @@
 from __future__ import division
 
+import os
+import signal
 import string
+from threading import Timer
 
 from six.moves import builtins
 from six.moves.urllib.parse import quote as urlquote
@@ -62,6 +65,12 @@ def _assert(x):
 env = website.env
 tell_sentry = website.tell_sentry
 
+if not website.db:
+    # Re-exec in 30 second to see if the DB is back up
+    # SIGTERM is used to tell gunicorn to gracefully stop the worker
+    # http://docs.gunicorn.org/en/stable/signals.html
+    Timer(30.0, lambda: os.kill(os.getpid(), signal.SIGTERM)).start()
+
 if env.cache_static:
     http_caching.compile_assets(website)
 elif env.clean_assets:
@@ -71,8 +80,8 @@ def _assert(x):
 # Periodic jobs
 # =============
 
-if env.run_cron_jobs:
-    conf = website.app_conf
+conf = website.app_conf
+if env.run_cron_jobs and conf:
     cron = Cron(website)
     cron(conf.update_global_stats_every, lambda: utils.update_global_stats(website))
     cron(conf.check_db_every, website.db.self_check, True)

liberapay/renderers/scss.py

@@ -17,7 +17,7 @@ def __init__(self, factory, *a, **kw):
         self.website = factory._configuration
         renderers.Renderer.__init__(self, factory, *a, **kw)
         self.cache_static = self.website.env.cache_static
-        compress = self.website.app_conf.compress_assets
+        compress = self.website.env.compress_assets
         output_style = 'compressed' if compress else 'nested'
         kw = dict(output_style=output_style)
         if self.website.project_root is not None:

liberapay/security/__init__.py

@@ -36,7 +36,7 @@ def set_default_security_headers(website, response, request=None):
             b"connect-src *;"  # for credit card data
             b"img-src *;"
             b"reflected-xss block;"
-        ) + website.app_conf.csp_extra.encode()
+        ) + website.env.csp_extra.encode()
         if website.canonical_scheme == 'https':
             csp += b"upgrade-insecure-requests;block-all-mixed-content;"
         response.headers[b'content-security-policy-report-only'] = csp

liberapay/security/authentication.py

@@ -103,6 +103,9 @@ def authenticate_user_if_possible(request, response, state, user, _):
     if request.line.uri.startswith('/assets/'):
         return
 
+    if not state['website'].db:
+        return
+
     # HTTP auth
     if b'Authorization' in request.headers:
         header = request.headers[b'Authorization']

liberapay/utils/i18n.py

@@ -108,6 +108,7 @@ def make_sorted_dict(keys, d):
     429: _("Too Many Requests"),
     500: _("Internal Server Error"),
     502: _("Upstream Error"),
+    503: _("Service Unavailable"),
 }
 del _
 

liberapay/wireup.py

@@ -5,7 +5,8 @@
 import logging
 import os
 import re
-from subprocess import check_call
+import signal
+from subprocess import call
 import traceback
 
 from six import text_type as str
@@ -23,6 +24,7 @@
 from liberapay import elsewhere
 import liberapay.billing.payday
 from liberapay.constants import CustomUndefined
+from liberapay.exceptions import NeedDatabase
 from liberapay.models.account_elsewhere import AccountElsewhere
 from liberapay.models.community import Community
 from liberapay.models.exchange_route import ExchangeRoute
@@ -51,25 +53,32 @@ def canonical(env):
     return locals()
 
 
-def database(env, tell_sentry, retry=True):
+class NoDB(object):
+
+    def __getattr__(self, attr):
+        raise NeedDatabase()
+
+    __bool__ = lambda self: False
+    __nonzero__ = __bool__
+
+    def register_model(self, model):
+        model.db = self
+
+
+def database(env, tell_sentry):
     dburl = env.database_url
     maxconn = env.database_maxconn
     try:
         db = DB(dburl, maxconn=maxconn)
-    except psycopg2.OperationalError:
+    except psycopg2.OperationalError as e:
+        tell_sentry(e, {})
         pg_dir = os.environ.get('OPENSHIFT_PG_DATA_DIR')
-        if not pg_dir:
-            # We're not in production, let the developer deal with it.
-            raise
-        if not retry:
-            # Give up
-            raise
-        try:
-            check_call(['pg_ctl', '-D', pg_dir, 'start', '-w', '-t', '120'])
-        except Exception as e:
-            tell_sentry(e, {})
-            raise
-        return database(env, tell_sentry, retry=False)
+        if pg_dir:
+            # We know where the postgres data is, try to start the server ourselves
+            r = call(['pg_ctl', '-D', pg_dir, 'start', '-w', '-t', '15'])
+            if r == 0:
+                return database(env, tell_sentry)
+        db = NoDB()
 
     for model in (AccountElsewhere, Community, ExchangeRoute, Participant):
         db.register_model(model)
@@ -90,8 +99,6 @@ class AppConf(object):
         bountysource_id=None.__class__,
         bountysource_secret=str,
         check_db_every=int,
-        compress_assets=bool,
-        csp_extra=str,
         dequeue_emails_every=int,
         facebook_callback=str,
         facebook_id=str,
@@ -159,11 +166,15 @@ def __init__(self, d):
 
 
 def app_conf(db):
+    if not db:
+        return {'app_conf': None}
     conf = AppConf(db.all("SELECT key, value FROM app_conf"))
     return {'app_conf': conf}
 
 
 def mail(app_conf, project_root='.'):
+    if not app_conf:
+        return
     smtp_conf = {
         k[5:]: v for k, v in app_conf.__dict__.items() if k.startswith('smtp_')
     }
@@ -192,6 +203,8 @@ def log_email(message):
 
 
 def billing(app_conf):
+    if not app_conf:
+        return
     from mangopaysdk.configuration import Configuration
     Configuration.BaseUrl = app_conf.mangopay_base_url
     Configuration.ClientID = app_conf.mangopay_client_id
@@ -217,6 +230,23 @@ def tell_sentry(exception, state, allow_reraise=True):
             # Only log server errors
             return
 
+        if isinstance(exception, NeedDatabase):
+            # Don't flood Sentry when DB is down
+            return
+
+        if isinstance(exception, psycopg2.Error):
+            from liberapay.website import website
+            if getattr(website, 'db', None):
+                try:
+                    website.db.one('SELECT 1 AS x')
+                except psycopg2.Error:
+                    # If it can't answer this simple query, it's down.
+                    website.db = NoDB()
+                    # Show the proper 503 error page
+                    state['exception'] = NeedDatabase()
+                    # Tell gunicorn to gracefully restart this worker
+                    os.kill(os.getpid(), signal.SIGTERM)
+
         if not sentry:
             if env.sentry_reraise and allow_reraise:
                 raise
@@ -266,6 +296,8 @@ def __iter__(self):
 
 
 def accounts_elsewhere(app_conf, asset):
+    if not app_conf:
+        return
     platforms = []
     for cls in elsewhere.CLASSES:
         conf = {
@@ -389,6 +421,8 @@ def env():
         DATABASE_MAXCONN=int,
         CANONICAL_HOST=str,
         CANONICAL_SCHEME=str,
+        COMPRESS_ASSETS=is_yesish,
+        CSP_EXTRA=str,
         SENTRY_DSN=str,
         SENTRY_RERAISE=is_yesish,
         GUNICORN_OPTS=str,

sql/app-conf-defaults.sql

@@ -8,8 +8,6 @@ INSERT INTO app_conf (key, value) VALUES
     ('bountysource_id', 'null'::jsonb),
     ('bountysource_secret', '""'::jsonb),
     ('check_db_every', '600'::jsonb),
-    ('compress_assets', 'false'::jsonb),
-    ('csp_extra', '""'::jsonb),
     ('dequeue_emails_every', '60'::jsonb),
     ('facebook_callback', '"http://localhost:8339/on/facebook/associate"'::jsonb),
     ('facebook_id', '"1418954898427187"'::jsonb),

sql/branch.sql

@@ -0,0 +1 @@
+DELETE FROM app_conf WHERE key in ('compress_assets', 'csp_extra');

style/base/base.scss

@@ -298,11 +298,7 @@ section.profile-statement {
     }
 }
 
-#notification-area {
-    bottom: 0;
-    left: 0;
-    position: fixed;
-    right: 0;
+#notification-area-bottom, #notification-area-top {
     text-align: center;
     z-index: 10;
 
@@ -322,6 +318,12 @@ section.profile-statement {
         color: $state-success-text;
     }
 }
+#notification-area-bottom {
+    bottom: 0;
+    left: 0;
+    position: fixed;
+    right: 0;
+}
 
 .dropdown-hover {
     display: inline-block;

templates/base.html

@@ -30,6 +30,13 @@
 % from 'templates/search.html' import search_form with context
 <body>
 
+% if not website.db
+<div id="notification-area-top"><div class="notification notification-error">{{ _(
+    "We're currently experiencing technical failures. As a result most things don't work. "
+    "Sorry for the inconvenience, we'll get everything back to normal ASAP."
+) }}</div></div>
+% endif
+
 <div id="wrapper">
     <nav class="navbar navbar-liberapay navbar-static-top">
         <div class="container">
@@ -109,7 +116,7 @@
 % set _success = request.qs.get('success')
 % if _success
 % set _success = _success and b64decode_s(_success, default=None)
-<div id="notification-area"><div class="notification notification-success">{{
+<div id="notification-area-bottom"><div class="notification notification-success">{{
     _success
 }}<span class="close">&times;</span></div></div>
 % endif

templates/no-db.html

@@ -0,0 +1,14 @@
+<h1>{{ _("Service Unavailable") }}</h1>
+
+<p>{{ _("We're unable to process your request right now, sorry.") }}</p>
+
+<p>{{ _("Please try again in a few minutes.") }}</p>
+
+% if request.method == 'GET'
+    <a class="btn btn-primary" href="{{ request.line.uri }}">{{ _("Try again") }}</a>
+% elif request.method == 'POST'
+    <form action="{{ request.line.uri }}" method="POST">
+        % include "templates/form-repost.html"
+        <button class="btn btn-primary">{{ _("Try again") }}</button>
+    </form>
+% endif

tests/py/test_pages.py

@@ -7,13 +7,15 @@
 import os
 import re
 
+from mock import patch
 from pando import json, Response
 import pytest
 
 from liberapay.billing.payday import Payday
 from liberapay.constants import SESSION
 from liberapay.testing.mangopay import MangopayHarness
 from liberapay.utils import b64encode_s, find_files
+from liberapay.wireup import NoDB
 
 
 overescaping_re = re.compile(r'&(#[0-9]{4}|[a-z]+);')
@@ -129,6 +131,26 @@ def test_github_associate(self):
     def test_twitter_associate(self):
         assert self.client.GxT('/on/twitter/associate').code == 400
 
+    def test_homepage_redirects_when_db_is_down(self):
+        with patch.multiple(self.website, db=NoDB()):
+            r = self.client.GET('/', raise_immediately=False)
+        assert r.code == 302
+        assert r.headers[b'Location'] == b'/about/'
+
+    def test_about_page_works_even_when_db_is_down(self):
+        alice = self.make_participant('alice')
+        with patch.multiple(self.website, db=NoDB()):
+            r = self.client.GET('/about/', auth_as=alice)
+        assert r.code == 200
+        assert b"Liberapay is " in r.body
+
+    def test_stats_page_is_503_when_db_is_down(self):
+        with patch.multiple(self.website, db=NoDB()):
+            r = self.client.GET('/about/stats', raise_immediately=False)
+        assert r.code == 503
+        assert b" technical failures." in r.body
+        assert b" unable to process your request " in r.body
+
     def test_paydays_json_gives_paydays(self):
         Payday.start()
         self.make_participant("alice")

www/index.html.spt

@@ -9,6 +9,9 @@ query_cache = QueryCache(website.db, threshold=5)
 
 [---]
 
+if not website.db:
+    response.redirect('/about/')
+
 sponsors = query_cache.all("""
     SELECT username, giving, avatar_url
       FROM ( SELECT * FROM sponsors ORDER BY random() * giving DESC LIMIT 10 ) foo