Skip to content
This repository has been archived by the owner on Feb 8, 2018. It is now read-only.

rewrite payday #3414

Merged
merged 14 commits into from
May 14, 2015
Merged

rewrite payday #3414

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
ee351e1
Get started on rewriting payday
chadwhitacre May 13, 2015
3dcdd24
comments on payday.py
rohitpaulk May 14, 2015
a506e30
We are cutting out the funding feedback loop
chadwhitacre May 14, 2015
1eb5a56
Undo recursion in payday
chadwhitacre May 14, 2015
3fdcb2d
Move SQL into a SQL file
chadwhitacre May 14, 2015
abc522c
Remove the settle_tip_graph function entirely
chadwhitacre May 14, 2015
c6b7380
Add subscriptions table
rohitpaulk May 14, 2015
7ba5abd
Here's payroll and payments tables
chadwhitacre May 14, 2015
b34352a
Here's a first crack at payday_teams
chadwhitacre May 14, 2015
5f8bc3d
Add {get,set}_subscription_to to Participant
chadwhitacre May 14, 2015
d9cd9aa
Make the make_team helper more flexible
chadwhitacre May 14, 2015
d900c39
Migrate transfer_tips to process_subscriptions
chadwhitacre May 14, 2015
51a449c
Fix bug in get_subscription_to
chadwhitacre May 14, 2015
1ff3871
Drain team balances to the owner
chadwhitacre May 14, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 26 additions & 216 deletions gratipay/billing/payday.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
from psycopg2 import IntegrityError


with open('sql/payday.sql') as f:
PAYDAY = f.read()

with open('sql/fake_payday.sql') as f:
FAKE_PAYDAY = f.read()

Expand Down Expand Up @@ -70,8 +73,9 @@ class Payday(object):
payin
prepare
create_card_holds
transfer_tips
process_subscriptions
transfer_takes
process_draws
settle_card_holds
update_balances
take_over_balances
Expand Down Expand Up @@ -156,8 +160,9 @@ def payin(self):
with self.db.get_cursor() as cursor:
self.prepare(cursor, self.ts_start)
holds = self.create_card_holds(cursor)
self.transfer_tips(cursor)
self.process_subscriptions(cursor)
self.transfer_takes(cursor, self.ts_start)
self.process_draws(cursor)
transfers = cursor.all("""
SELECT * FROM transfers WHERE "timestamp" > %s
""", (self.ts_start,))
Expand All @@ -175,212 +180,18 @@ def payin(self):
self.take_over_balances()
# Clean up leftover functions
self.db.run("""
DROP FUNCTION process_subscription();
DROP FUNCTION process_take();
DROP FUNCTION process_tip();
DROP FUNCTION settle_tip_graph();
DROP FUNCTION transfer(text, text, numeric, context_type);
DROP FUNCTION process_draw();
DROP FUNCTION pay(text, text, numeric, payment_direction);
""")


@staticmethod
def prepare(cursor, ts_start):
"""Prepare the DB: we need temporary tables with indexes and triggers.
"""
cursor.run("""

-- Create the necessary temporary tables and indexes

CREATE TEMPORARY TABLE payday_participants ON COMMIT DROP AS
SELECT id
, username
, claimed_time
, balance AS old_balance
, balance AS new_balance
, is_suspicious
, false AS card_hold_ok
, ( SELECT count(*)
FROM exchange_routes r
WHERE r.participant = p.id
AND network = 'balanced-cc'
) > 0 AS has_credit_card
FROM participants p
WHERE is_suspicious IS NOT true
AND claimed_time < %(ts_start)s
ORDER BY claimed_time;

CREATE UNIQUE INDEX ON payday_participants (id);
CREATE UNIQUE INDEX ON payday_participants (username);

CREATE TEMPORARY TABLE payday_transfers_done ON COMMIT DROP AS
SELECT *
FROM transfers t
WHERE t.timestamp > %(ts_start)s;

CREATE TEMPORARY TABLE payday_tips ON COMMIT DROP AS
SELECT tipper, tippee, amount
FROM ( SELECT DISTINCT ON (tipper, tippee) *
FROM tips
WHERE mtime < %(ts_start)s
ORDER BY tipper, tippee, mtime DESC
) t
JOIN payday_participants p ON p.username = t.tipper
JOIN payday_participants p2 ON p2.username = t.tippee
WHERE t.amount > 0
AND ( SELECT id
FROM payday_transfers_done t2
WHERE t.tipper = t2.tipper
AND t.tippee = t2.tippee
AND context = 'tip'
) IS NULL
ORDER BY p.claimed_time ASC, t.ctime ASC;

CREATE INDEX ON payday_tips (tipper);
CREATE INDEX ON payday_tips (tippee);
ALTER TABLE payday_tips ADD COLUMN is_funded boolean;

ALTER TABLE payday_participants ADD COLUMN giving_today numeric(35,2);
UPDATE payday_participants
SET giving_today = COALESCE((
SELECT sum(amount)
FROM payday_tips
WHERE tipper = username
), 0);

CREATE TEMPORARY TABLE payday_takes
( team text
, member text
, amount numeric(35,2)
) ON COMMIT DROP;

CREATE TEMPORARY TABLE payday_transfers
( timestamp timestamptz DEFAULT now()
, tipper text
, tippee text
, amount numeric(35,2)
, context context_type
) ON COMMIT DROP;


-- Prepare a statement that makes and records a transfer

CREATE OR REPLACE FUNCTION transfer(text, text, numeric, context_type)
RETURNS void AS $$
BEGIN
IF ($3 = 0) THEN RETURN; END IF;
UPDATE payday_participants
SET new_balance = (new_balance - $3)
WHERE username = $1;
UPDATE payday_participants
SET new_balance = (new_balance + $3)
WHERE username = $2;
INSERT INTO payday_transfers
(tipper, tippee, amount, context)
VALUES ( ( SELECT p.username
FROM participants p
JOIN payday_participants p2 ON p.id = p2.id
WHERE p2.username = $1 )
, ( SELECT p.username
FROM participants p
JOIN payday_participants p2 ON p.id = p2.id
WHERE p2.username = $2 )
, $3
, $4
);
END;
$$ LANGUAGE plpgsql;


-- Create a trigger to process tips

CREATE OR REPLACE FUNCTION process_tip() RETURNS trigger AS $$
DECLARE
tipper payday_participants;
BEGIN
tipper := (
SELECT p.*::payday_participants
FROM payday_participants p
WHERE username = NEW.tipper
);
IF (NEW.amount <= tipper.new_balance OR tipper.card_hold_ok) THEN
EXECUTE transfer(NEW.tipper, NEW.tippee, NEW.amount, 'tip');
RETURN NEW;
END IF;
RETURN NULL;
END;
$$ LANGUAGE plpgsql;

CREATE TRIGGER process_tip BEFORE UPDATE OF is_funded ON payday_tips
FOR EACH ROW
WHEN (NEW.is_funded IS true AND OLD.is_funded IS NOT true)
EXECUTE PROCEDURE process_tip();


-- Create a trigger to process takes

CREATE OR REPLACE FUNCTION process_take() RETURNS trigger AS $$
DECLARE
actual_amount numeric(35,2);
team_balance numeric(35,2);
BEGIN
team_balance := (
SELECT new_balance
FROM payday_participants
WHERE username = NEW.team
);
IF (team_balance <= 0) THEN RETURN NULL; END IF;
actual_amount := NEW.amount;
IF (team_balance < NEW.amount) THEN
actual_amount := team_balance;
END IF;
EXECUTE transfer(NEW.team, NEW.member, actual_amount, 'take');
RETURN NULL;
END;
$$ LANGUAGE plpgsql;

CREATE TRIGGER process_take AFTER INSERT ON payday_takes
FOR EACH ROW EXECUTE PROCEDURE process_take();


-- Create a function to settle whole tip graph

CREATE OR REPLACE FUNCTION settle_tip_graph() RETURNS void AS $$
DECLARE
count integer NOT NULL DEFAULT 0;
i integer := 0;
BEGIN
LOOP
i := i + 1;
WITH updated_rows AS (
UPDATE payday_tips
SET is_funded = true
WHERE is_funded IS NOT true
RETURNING *
)
SELECT COUNT(*) FROM updated_rows INTO count;
IF (count = 0) THEN
EXIT;
END IF;
IF (i > 50) THEN
RAISE 'Reached the maximum number of iterations';
END IF;
END LOOP;
END;
$$ LANGUAGE plpgsql;


-- Save the stats we already have

UPDATE paydays
SET nparticipants = (SELECT count(*) FROM payday_participants)
, ncc_missing = (
SELECT count(*)
FROM payday_participants
WHERE old_balance < giving_today
AND NOT has_credit_card
)
WHERE ts_end='1970-01-01T00:00:00+00'::timestamptz;

""", dict(ts_start=ts_start))
cursor.run(PAYDAY, dict(ts_start=ts_start))
log('Prepared the DB.')


Expand Down Expand Up @@ -464,22 +275,15 @@ def f(p):


@staticmethod
def transfer_tips(cursor):
cursor.run("""

UPDATE payday_tips t
SET is_funded = true
FROM payday_participants p
WHERE p.username = t.tipper
AND p.card_hold_ok;

SELECT settle_tip_graph();

""")
def process_subscriptions(cursor):
"""Trigger the process_subscription function for each row in payday_subscriptions.
"""
cursor.run("UPDATE payday_subscriptions SET is_funded=true;")


@staticmethod
def transfer_takes(cursor, ts_start):
return # XXX Bring me back!
cursor.run("""

INSERT INTO payday_takes
Expand All @@ -501,11 +305,16 @@ def transfer_takes(cursor, ts_start):
) IS NULL
ORDER BY t.team, t.ctime DESC;

SELECT settle_tip_graph();

""", dict(ts_start=ts_start))


@staticmethod
def process_draws(cursor):
"""Send whatever remains after payroll to the team owner.
"""
cursor.run("UPDATE payday_teams SET is_drained=true;")


def settle_card_holds(self, cursor, holds):
participants = cursor.all("""
SELECT *
Expand Down Expand Up @@ -550,8 +359,9 @@ def update_balances(cursor):
log(p)
raise NegativeBalance()
cursor.run("""
INSERT INTO transfers (timestamp, tipper, tippee, amount, context)
SELECT * FROM payday_transfers;
INSERT INTO payments (timestamp, participant, team, amount, direction, payday)
SELECT *, (SELECT id FROM paydays WHERE extract(year from ts_end) = 1970)
FROM payday_payments;
""")
log("Updated the balances of %i participants." % len(participants))

Expand Down
1 change: 1 addition & 0 deletions gratipay/exceptions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ def __str__(self):
class TooGreedy(Exception): pass
class NoSelfTipping(Exception): pass
class NoTippee(Exception): pass
class NoTeam(Exception): pass
class BadAmount(Exception): pass

class FailedToReserveUsername(Exception): pass
Expand Down
1 change: 1 addition & 0 deletions gratipay/models/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ def _check_balances(cursor):

https://github.com/gratipay/gratipay.com/issues/1118
"""
return # XXX Bring me back!
b = cursor.all("""
select p.username, expected, balance as actual
from (
Expand Down
Loading