Merge pull request #779 from liberapay/currencies-4

Currencies - part 4

js/charts.js

@@ -20,6 +20,10 @@ Liberapay.charts.make = function(series, button) {
         return;
     }
 
+    function parsePoint(o) {
+        return parseFloat(o.amount || o);
+    }
+
     // Reverse the series.
     // ===================
     // For historical reasons the API is descending when we want ascending.
@@ -53,7 +57,7 @@ Liberapay.charts.make = function(series, button) {
 
     var maxes = charts.map(function(chart) {
         return series.reduce(function(previous, current) {
-            return Math.max(previous, current[chart.data('chart')]);
+            return Math.max(previous, parsePoint(current[chart.data('chart')]));
         }, 0);
     });
 
@@ -67,7 +71,7 @@ Liberapay.charts.make = function(series, button) {
     charts.forEach(function(chart, chart_index) {
         chart.css('min-width', (series.length * 5) + 'px');
         series.forEach(function(point, index) {
-            var y = parseFloat(point[chart.data('chart')]);
+            var y = parsePoint(point[chart.data('chart')]);
             var bar = $('<div>').addClass('bar');
             var shaded = $('<div>').addClass('shaded');
             shaded.html('<span class="y-label">'+ y.toFixed() +'</span>');

liberapay/billing/payday.py

@@ -118,10 +118,10 @@ def shuffle(self, log_dir='.'):
                  , w.remote_id AS tipper_wallet_id
                  , w2.remote_id AS tippee_wallet_id
               FROM payday_transfers t
-              JOIN wallets w ON w.owner = t.tipper AND
+         LEFT JOIN wallets w ON w.owner = t.tipper AND
                    w.balance::currency = t.amount::currency AND
                    w.is_current IS TRUE
-              JOIN wallets w2 ON w2.owner = t.tippee AND
+         LEFT JOIN wallets w2 ON w2.owner = t.tippee AND
                    w2.balance::currency = t.amount::currency AND
                    w2.is_current IS TRUE
           ORDER BY t.id
@@ -256,14 +256,14 @@ def prepare(cursor, ts_start):
 
         CREATE OR REPLACE FUNCTION process_tip() RETURNS trigger AS $$
             DECLARE
-                tipper payday_participants;
+                tipper_balances currency_basket;
             BEGIN
-                tipper := (
-                    SELECT p.*::payday_participants
+                tipper_balances := (
+                    SELECT balances
                       FROM payday_participants p
                      WHERE id = NEW.tipper
                 );
-                IF (tipper.balances >= NEW.amount) THEN
+                IF (tipper_balances >= NEW.amount) THEN
                     EXECUTE transfer(NEW.tipper, NEW.tippee, NEW.amount, 'tip', NULL, NULL);
                     RETURN NEW;
                 END IF;
@@ -334,14 +334,14 @@ def transfer_takes(cursor, team_id, currency):
                AND tippee = %(team_id)s
                AND p.balances >= amount
          RETURNING t.id, t.tipper, t.amount AS full_amount
-                 , COALESCE((
-                       SELECT sum(tr.amount)
+                 , coalesce_currency_amount((
+                       SELECT sum(tr.amount, t.amount::currency)
                          FROM transfers tr
                         WHERE tr.tipper = t.tipper
                           AND tr.team = %(team_id)s
                           AND tr.context = 'take'
                           AND tr.status = 'succeeded'
-                   ), zero(t.amount)) AS past_transfers_sum
+                   ), t.amount::currency) AS past_transfers_sum
         """, args)]
         takes = [NS(t._asdict()) for t in cursor.all("""
             SELECT t.member, t.amount
@@ -525,7 +525,7 @@ def settle_debts(db):
                       JOIN wallets w_debtor ON w_debtor.owner = d.debtor AND
                            w_debtor.balance::currency = d.amount::currency AND
                            w_debtor.is_current IS TRUE
-                      JOIN wallets w_creditor ON w_creditor.owner = d.creditor AND
+                 LEFT JOIN wallets w_creditor ON w_creditor.owner = d.creditor AND
                            w_creditor.balance::currency = d.amount::currency AND
                            w_creditor.is_current IS TRUE
                       JOIN participants p_creditor ON p_creditor.id = d.creditor
@@ -704,12 +704,12 @@ def update_cached_amounts(cls):
             UPDATE participants p
                SET giving = p2.giving
               FROM ( SELECT p2.id
-                          , COALESCE((
+                          , coalesce_currency_amount((
                                 SELECT sum(amount, p2.main_currency)
                                   FROM payday_tips t
                                  WHERE t.tipper = p2.id
                                    AND t.is_funded
-                            ), zero(p2.giving)) AS giving
+                            ), p2.main_currency) AS giving
                        FROM participants p2
                    ) p2
              WHERE p.id = p2.id
@@ -718,12 +718,12 @@ def update_cached_amounts(cls):
             UPDATE participants p
                SET taking = p2.taking
               FROM ( SELECT p2.id
-                          , COALESCE((
+                          , coalesce_currency_amount((
                                 SELECT sum(t.amount, p2.main_currency)
                                   FROM payday_transfers t
                                  WHERE t.tippee = p2.id
                                    AND context = 'take'
-                            ), zero(p2.taking)) AS taking
+                            ), p2.main_currency) AS taking
                        FROM participants p2
                    ) p2
              WHERE p.id = p2.id
@@ -732,12 +732,12 @@ def update_cached_amounts(cls):
             UPDATE participants p
                SET receiving = p2.receiving
               FROM ( SELECT p2.id
-                          , p2.taking + COALESCE((
+                          , p2.taking + coalesce_currency_amount((
                                 SELECT sum(amount, p2.main_currency)
                                   FROM payday_tips t
                                  WHERE t.tippee = p2.id
                                    AND t.is_funded
-                            ), zero(p2.taking)) AS receiving
+                            ), p2.main_currency) AS receiving
                        FROM participants p2
                    ) p2
              WHERE p.id = p2.id
@@ -747,12 +747,12 @@ def update_cached_amounts(cls):
             UPDATE participants p
                SET leftover = p2.leftover
               FROM ( SELECT p2.id
-                          , p2.receiving - COALESCE((
+                          , p2.receiving - coalesce_currency_amount((
                                 SELECT sum(t.amount, p2.main_currency)
                                   FROM payday_transfers t
                                  WHERE t.tippee = p2.id
                                     OR t.team = p2.id
-                            ), zero(p2.receiving)) AS leftover
+                            ), p2.main_currency) AS leftover
                        FROM participants p2
                    ) p2
              WHERE p.id = p2.id
@@ -847,7 +847,7 @@ def notify_participants(self):
 
         # Identity-required notifications
         participants = self.db.all("""
-            SELECT p.*::participants
+            SELECT p
               FROM participants p
              WHERE mangopay_user_id IS NULL
                AND kind IN ('individual', 'organization')
@@ -858,7 +858,7 @@ def notify_participants(self):
                        JOIN participants p2 ON p2.id = t.tipper
                       WHERE t.tippee = p.id
                         AND t.amount > 0
-                        AND p2.balance > t.amount
+                        AND t.is_funded
                    )
         """)
         for p in participants:
@@ -877,8 +877,8 @@ def notify_participants(self):
                    GROUP BY t.tipper, t.amount::currency
                    ) a
               JOIN participants p ON p.id = a.tipper
-              JOIN wallets w ON w.owner = p.id AND w.balance::currency = needed::currency
-             WHERE w.balance < needed
+         LEFT JOIN wallets w ON w.owner = p.id AND w.balance::currency = needed::currency
+             WHERE COALESCE(w.balance, zero(needed)) < needed
                AND EXISTS (
                      SELECT 1
                        FROM transfers t

liberapay/billing/transactions.py

@@ -438,13 +438,11 @@ def propagate_exchange(cursor, participant, exchange, error, amount):
 
 
 def transfer(db, tipper, tippee, amount, context, **kw):
-    try:
-        tipper_wallet = NS(dict(remote_id=kw['tipper_wallet_id'], remote_owner_id=kw['tipper_mango_id']))
-    except KeyError:
+    tipper_wallet = NS(remote_id=kw.get('tipper_wallet_id'), remote_owner_id=kw.get('tipper_mango_id'))
+    if not all(tipper_wallet.__dict__.values()):
         tipper_wallet = Participant.from_id(tipper).get_current_wallet(amount.currency)
-    try:
-        tippee_wallet = NS(dict(remote_id=kw['tippee_wallet_id'], remote_owner_id=kw['tippee_mango_id']))
-    except KeyError:
+    tippee_wallet = NS(remote_id=kw.get('tippee_wallet_id'), remote_owner_id=kw.get('tippee_mango_id'))
+    if not all(tippee_wallet.__dict__.values()):
         tippee_wallet = Participant.from_id(tippee).get_current_wallet(amount.currency, create=True)
     wallet_from = tipper_wallet.remote_id
     wallet_to = tippee_wallet.remote_id

liberapay/cron.py

@@ -61,6 +61,11 @@ def f():
                         func()
                     except Exception as e:
                         self.website.tell_sentry(e, {})
+                        # retry in 5 minutes
+                        sleep(300)
+                    else:
+                        # success, sleep until tomorrow
+                        sleep(3600 * 23)
             else:
                 while True:
                     try:

liberapay/models/_mixin_team.py

@@ -191,14 +191,14 @@ def recompute_actual_takes(self, cursor, member=None):
         from liberapay.billing.payday import Payday
         tips = [NS(t._asdict()) for t in cursor.all("""
             SELECT t.id, t.tipper, t.amount AS full_amount
-                 , COALESCE((
-                       SELECT sum(tr.amount)
+                 , coalesce_currency_amount((
+                       SELECT sum(tr.amount, t.amount::currency)
                          FROM transfers tr
                         WHERE tr.tipper = t.tipper
                           AND tr.team = %(team_id)s
                           AND tr.context = 'take'
                           AND tr.status = 'succeeded'
-                   ), zero(t.amount)) AS past_transfers_sum
+                   ), t.amount::currency) AS past_transfers_sum
               FROM current_tips t
               JOIN participants p ON p.id = t.tipper
              WHERE t.tippee = %(team_id)s

liberapay/models/participant.py

@@ -1569,7 +1569,7 @@ def update_giving(self, cursor=None):
         # Update giving on participant
         giving = (cursor or self.db).one("""
             UPDATE participants p
-               SET giving = COALESCE((
+               SET giving = coalesce_currency_amount((
                      SELECT sum(amount, %(currency)s)
                        FROM current_tips
                        JOIN participants p2 ON p2.id = tippee
@@ -1578,7 +1578,7 @@ def update_giving(self, cursor=None):
                         AND (p2.mangopay_user_id IS NOT NULL OR kind = 'group')
                         AND amount > 0
                         AND is_funded
-                   ), zero(p.giving))
+                   ), p.main_currency)
              WHERE p.id = %(id)s
          RETURNING giving
         """, dict(id=self.id, currency=self.main_currency))
@@ -1600,9 +1600,9 @@ def update_receiving(self, cursor=None):
                             AND is_funded
                      )
                 UPDATE participants p
-                   SET receiving = (
-                           COALESCE((SELECT sum(amount, %(currency)s) FROM our_tips), %(zero)s) +
-                           COALESCE(taking, %(zero)s)
+                   SET receiving = taking + coalesce_currency_amount(
+                           (SELECT sum(amount, %(currency)s) FROM our_tips),
+                           %(currency)s
                        )
                      , npatrons = COALESCE((SELECT count(*) FROM our_tips), 0)
                  WHERE p.id = %(id)s

liberapay/utils/__init__.py

@@ -32,8 +32,8 @@
 
 
 class NS(object):
-    def __init__(self, d):
-        self.__dict__.update(d)
+    def __init__(self, *d, **kw):
+        self.__dict__.update(*d, **kw)
 
 
 def get_participant(state, restrict=True, redirect_stub=True, allow_member=False,

liberapay/utils/history.py

@@ -4,6 +4,7 @@
 
 from ..website import website
 from .currencies import MoneyBasket
+from . import group_by
 
 
 def get_end_of_year_balances(db, participant, year, current_year):
@@ -100,19 +101,25 @@ def iter_payday_events(db, participant, year=None):
         successes = [t for t in transfers if t['status'] == 'succeeded' and not t['refund_ref']]
         regular_donations = [t for t in successes if t['context'] in ('tip', 'take')]
         reimbursements = [t for t in successes if t['context'] == 'expense']
+        regular_donations_by_currency = group_by(regular_donations, lambda t: t['amount'].currency)
+        reimbursements_by_currency = group_by(reimbursements, lambda t: t['amount'].currency)
         yield dict(
             kind='totals',
             regular_donations=dict(
                 sent=MoneyBasket.sum(t['amount'] for t in regular_donations if t['tipper'] == id),
                 received=MoneyBasket.sum(t['amount'] for t in regular_donations if t['tippee'] == id),
-                npatrons=len(set(t['tipper'] for t in regular_donations if t['tippee'] == id)),
-                ntippees=len(set(t['tippee'] for t in regular_donations if t['tipper'] == id)),
+                npatrons={k: len(set(t['tipper'] for t in transfers if t['tippee'] == id))
+                          for k, transfers in regular_donations_by_currency.items()},
+                ntippees={k: len(set(t['tippee'] for t in transfers if t['tipper'] == id))
+                          for k, transfers in regular_donations_by_currency.items()},
             ),
             reimbursements=dict(
                 sent=MoneyBasket.sum(t['amount'] for t in reimbursements if t['tipper'] == id),
                 received=MoneyBasket.sum(t['amount'] for t in reimbursements if t['tippee'] == id),
-                npayers=len(set(t['tipper'] for t in reimbursements if t['tippee'] == id)),
-                nrecipients=len(set(t['tippee'] for t in reimbursements if t['tipper'] == id)),
+                npayers={k: len(set(t['tipper'] for t in transfers if t['tippee'] == id))
+                         for k, transfers in reimbursements_by_currency.items()},
+                nrecipients={k: len(set(t['tippee'] for t in transfers if t['tipper'] == id))
+                             for k, transfers in reimbursements_by_currency.items()},
             ),
         )
         del successes, regular_donations, reimbursements

liberapay/wireup.py

@@ -106,7 +106,7 @@ def adapt_money(m):
     register_adapter(Money, adapt_money)
 
     def cast_currency_amount(v, cursor):
-        return None if v is None else Money(*v[1:-1].split(','))
+        return None if v in (None, '(,)') else Money(*v[1:-1].split(','))
     try:
         oid = db.one("SELECT 'currency_amount'::regtype::oid")
         register_type(new_type((oid,), _str('currency_amount'), cast_currency_amount))

sql/branch.sql

@@ -0,0 +1,19 @@
+BEGIN;
+    CREATE FUNCTION coalesce_currency_amount(currency_amount, currency) RETURNS currency_amount AS $$
+        BEGIN RETURN (COALESCE($1.amount, '0.00'::numeric), COALESCE($1.currency, $2)); END;
+    $$ LANGUAGE plpgsql IMMUTABLE;
+    CREATE OR REPLACE FUNCTION initialize_amounts() RETURNS trigger AS $$
+        BEGIN
+            NEW.giving = coalesce_currency_amount(NEW.giving, NEW.main_currency);
+            NEW.receiving = coalesce_currency_amount(NEW.receiving, NEW.main_currency);
+            NEW.taking = coalesce_currency_amount(NEW.taking, NEW.main_currency);
+            NEW.leftover = coalesce_currency_amount(NEW.leftover, NEW.main_currency);
+            NEW.balance = coalesce_currency_amount(NEW.balance, NEW.main_currency);
+            RETURN NEW;
+        END;
+    $$ LANGUAGE plpgsql;
+    DROP TRIGGER initialize_amounts ON participants;
+    CREATE TRIGGER initialize_amounts
+        BEFORE INSERT OR UPDATE ON participants
+        FOR EACH ROW EXECUTE PROCEDURE initialize_amounts();
+END;