diff --git a/gratipay/models/participant/__init__.py b/gratipay/models/participant/__init__.py index 2bd9beddec..ec640a5d03 100644 --- a/gratipay/models/participant/__init__.py +++ b/gratipay/models/participant/__init__.py @@ -2,17 +2,13 @@ """ from __future__ import print_function, unicode_literals -from datetime import timedelta from decimal import Decimal -import pickle -from time import sleep import uuid from aspen.utils import utcnow import balanced import braintree from dependency_injection import resolve_dependencies -from markupsafe import escape as htmlescape from postgres.orm import Model from psycopg2 import IntegrityError @@ -25,11 +21,6 @@ UsernameIsRestricted, UsernameAlreadyTaken, BadAmount, - EmailAlreadyTaken, - CannotRemovePrimaryEmail, - EmailNotVerified, - TooManyEmailAddresses, - ResendingTooFast, ) from gratipay.billing.instruments import CreditCard @@ -38,19 +29,17 @@ from gratipay.models.exchange_route import ExchangeRoute from gratipay.models.team import Team from gratipay.models.team.takes import ZERO -from gratipay.security.crypto import constant_time_compare from gratipay.utils import ( i18n, is_card_expiring, - emails, markdown, notifications, pricing, - encode_for_querystring, ) from gratipay.utils.username import safely_reserve_a_username from .identity import Identity +from .email import Email ASCII_ALLOWED_IN_USERNAME = set("0123456789" "abcdefghijklmnopqrstuvwxyz" @@ -58,12 +47,10 @@ ".,-_:@ ") # We use | in Sentry logging, so don't make that allowable. :-) -EMAIL_HASH_TIMEOUT = timedelta(hours=24) - USERNAME_MAX_SIZE = 32 -class Participant(Model, Identity): +class Participant(Model, Email, Identity): """Represent a Gratipay participant. """ @@ -378,230 +365,6 @@ def clear_personal_information(self, cursor): self.set_attributes(**r._asdict()) - # Emails - # ====== - - def add_email(self, email, resend_threshold='3 minutes'): - """ - This is called when - 1) Adding a new email address - 2) Resending the verification email for an unverified email address - - Returns the number of emails sent. - """ - - # Check that this address isn't already verified - owner = self.db.one(""" - SELECT p.username - FROM emails e INNER JOIN participants p - ON e.participant_id = p.id - WHERE e.address = %(email)s - AND e.verified IS true - """, locals()) - if owner: - if owner == self.username: - return 0 - else: - raise EmailAlreadyTaken(email) - - if len(self.get_emails()) > 9: - raise TooManyEmailAddresses(email) - - nonce = str(uuid.uuid4()) - verification_start = utcnow() - - nrecent = self.db.one( "SELECT count(*) FROM emails WHERE address=%s AND " - "%s - verification_start < %s" - , (email, verification_start, resend_threshold) - ) - if nrecent: - raise ResendingTooFast() - - try: - with self.db.get_cursor() as c: - add_event(c, 'participant', dict(id=self.id, action='add', values=dict(email=email))) - c.run(""" - INSERT INTO emails - (address, nonce, verification_start, participant_id) - VALUES (%s, %s, %s, %s) - """, (email, nonce, verification_start, self.id)) - except IntegrityError: - nonce = self.db.one(""" - UPDATE emails - SET verification_start=%s - WHERE participant_id=%s - AND address=%s - AND verified IS NULL - RETURNING nonce - """, (verification_start, self.id, email)) - if not nonce: - return self.add_email(email) - - base_url = gratipay.base_url - username = self.username_lower - encoded_email = encode_for_querystring(email) - link = "{base_url}/~{username}/emails/verify.html?email2={encoded_email}&nonce={nonce}" - r = self.send_email('verification', - email=email, - link=link.format(**locals()), - include_unsubscribe=False) - assert r == 1 # Make sure the verification email was sent - if self.email_address: - self.send_email('verification_notice', - new_email=email, - include_unsubscribe=False) - return 2 - return 1 - - def update_email(self, email): - if not getattr(self.get_email(email), 'verified', False): - raise EmailNotVerified(email) - username = self.username - with self.db.get_cursor() as c: - add_event(c, 'participant', dict(id=self.id, action='set', values=dict(primary_email=email))) - c.run(""" - UPDATE participants - SET email_address=%(email)s - WHERE username=%(username)s - """, locals()) - self.set_attributes(email_address=email) - - def verify_email(self, email, nonce): - if '' in (email, nonce): - return emails.VERIFICATION_MISSING - r = self.get_email(email) - if r is None: - return emails.VERIFICATION_FAILED - if r.verified: - assert r.nonce is None # and therefore, order of conditions matters - return emails.VERIFICATION_REDUNDANT - if not constant_time_compare(r.nonce, nonce): - return emails.VERIFICATION_FAILED - if (utcnow() - r.verification_start) > EMAIL_HASH_TIMEOUT: - return emails.VERIFICATION_EXPIRED - try: - self.db.run(""" - UPDATE emails - SET verified=true, verification_end=now(), nonce=NULL - WHERE participant_id=%s - AND address=%s - AND verified IS NULL - """, (self.id, email)) - except IntegrityError: - return emails.VERIFICATION_STYMIED - - if not self.email_address: - self.update_email(email) - return emails.VERIFICATION_SUCCEEDED - - def get_email(self, email): - return self.db.one(""" - SELECT * - FROM emails - WHERE participant_id=%s - AND address=%s - """, (self.id, email)) - - def get_emails(self): - return self.db.all(""" - SELECT * - FROM emails - WHERE participant_id=%s - ORDER BY id - """, (self.id,)) - - def get_verified_email_addresses(self): - return [email.address for email in self.get_emails() if email.verified] - - def remove_email(self, address): - if address == self.email_address: - raise CannotRemovePrimaryEmail() - with self.db.get_cursor() as c: - add_event(c, 'participant', dict(id=self.id, action='remove', values=dict(email=address))) - c.run("DELETE FROM emails WHERE participant_id=%s AND address=%s", - (self.id, address)) - - def send_email(self, spt_name, **context): - context['participant'] = self - context['username'] = self.username - context['button_style'] = ( - "color: #fff; text-decoration:none; display:inline-block; " - "padding: 0 15px; background: #396; white-space: nowrap; " - "font: normal 14px/40px Arial, sans-serif; border-radius: 3px" - ) - context.setdefault('include_unsubscribe', True) - email = context.setdefault('email', self.email_address) - if not email: - return 0 # Not Sent - langs = i18n.parse_accept_lang(self.email_lang or 'en') - locale = i18n.match_lang(langs) - i18n.add_helpers_to_context(self._tell_sentry, context, locale) - context['escape'] = lambda s: s - context_html = dict(context) - i18n.add_helpers_to_context(self._tell_sentry, context_html, locale) - context_html['escape'] = htmlescape - spt = self._emails[spt_name] - base_spt = self._emails['base'] - def render(t, context): - b = base_spt[t].render(context).strip() - return b.replace('$body', spt[t].render(context).strip()) - - message = {} - message['Source'] = 'Gratipay Support ' - message['Destination'] = {} - message['Destination']['ToAddresses'] = ["%s <%s>" % (self.username, email)] # "Name " - message['Message'] = {} - message['Message']['Subject'] = {} - message['Message']['Subject']['Data'] = spt['subject'].render(context).strip() - message['Message']['Body'] = { - 'Text': { - 'Data': render('text/plain', context) - }, - 'Html': { - 'Data': render('text/html', context_html) - } - } - - self._mailer.send_email(**message) - return 1 # Sent - - def queue_email(self, spt_name, **context): - self.db.run(""" - INSERT INTO email_queue - (participant, spt_name, context) - VALUES (%s, %s, %s) - """, (self.id, spt_name, pickle.dumps(context))) - - @classmethod - def dequeue_emails(cls): - fetch_messages = lambda: cls.db.all(""" - SELECT * - FROM email_queue - ORDER BY id ASC - LIMIT 60 - """) - nsent = 0 - while True: - messages = fetch_messages() - if not messages: - break - for msg in messages: - p = cls.from_id(msg.participant) - r = p.send_email(msg.spt_name, **pickle.loads(msg.context)) - cls.db.run("DELETE FROM email_queue WHERE id = %s", (msg.id,)) - if r == 1: - sleep(1) - nsent += r - return nsent - - def set_email_lang(self, accept_lang): - if not accept_lang: - return - self.db.run("UPDATE participants SET email_lang=%s WHERE id=%s", - (accept_lang, self.id)) - self.set_attributes(email_lang=accept_lang) - - # Notifications # ============= diff --git a/gratipay/models/participant/email.py b/gratipay/models/participant/email.py new file mode 100644 index 0000000000..abb578c50e --- /dev/null +++ b/gratipay/models/participant/email.py @@ -0,0 +1,292 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, division, print_function, unicode_literals + +import pickle +import uuid +from datetime import timedelta +from time import sleep + +from aspen.utils import utcnow +from markupsafe import escape as htmlescape +from psycopg2 import IntegrityError + +import gratipay +from gratipay.exceptions import EmailAlreadyTaken, CannotRemovePrimaryEmail, EmailNotVerified +from gratipay.exceptions import TooManyEmailAddresses, ResendingTooFast +from gratipay.models import add_event +from gratipay.security.crypto import constant_time_compare +from gratipay.utils import emails, encode_for_querystring, i18n + + +EMAIL_HASH_TIMEOUT = timedelta(hours=24) + + +class Email(object): + """Participants may associate email addresses with their account. + + Email addresses are stored in an ``emails`` table in the database, which + holds the addresses themselves as well as info related to address + verification. While a participant may have multiple email addresses on + file, verified or not, only one will be the *primary* email address: the + one also recorded in ``participants.email_address``. It's a bug for the + primary address not to be verified, or for an address to be in + ``participants.email_address`` but not also in ``emails``. + + Having a verified email is a prerequisite for certain other features on + Gratipay, such as linking a PayPal account, or filing a national identity. + + """ + + def add_email(self, email, resend_threshold='3 minutes'): + """Add an email address for a participant. + + This is called when adding a new email address, and when resending the + verification email for an unverified email address. + + :returns: the number of emails sent. + + """ + + # Check that this address isn't already verified + owner = self.db.one(""" + SELECT p.username + FROM emails e INNER JOIN participants p + ON e.participant_id = p.id + WHERE e.address = %(email)s + AND e.verified IS true + """, locals()) + if owner: + if owner == self.username: + return 0 + else: + raise EmailAlreadyTaken(email) + + if len(self.get_emails()) > 9: + raise TooManyEmailAddresses(email) + + nonce = str(uuid.uuid4()) + verification_start = utcnow() + + nrecent = self.db.one( "SELECT count(*) FROM emails WHERE address=%s AND " + "%s - verification_start < %s" + , (email, verification_start, resend_threshold) + ) + if nrecent: + raise ResendingTooFast() + + try: + with self.db.get_cursor() as c: + add_event(c, 'participant', dict(id=self.id, action='add', values=dict(email=email))) + c.run(""" + INSERT INTO emails + (address, nonce, verification_start, participant_id) + VALUES (%s, %s, %s, %s) + """, (email, nonce, verification_start, self.id)) + except IntegrityError: + nonce = self.db.one(""" + UPDATE emails + SET verification_start=%s + WHERE participant_id=%s + AND address=%s + AND verified IS NULL + RETURNING nonce + """, (verification_start, self.id, email)) + if not nonce: + return self.add_email(email) + + base_url = gratipay.base_url + username = self.username_lower + encoded_email = encode_for_querystring(email) + link = "{base_url}/~{username}/emails/verify.html?email2={encoded_email}&nonce={nonce}" + r = self.send_email('verification', + email=email, + link=link.format(**locals()), + include_unsubscribe=False) + assert r == 1 # Make sure the verification email was sent + if self.email_address: + self.send_email('verification_notice', + new_email=email, + include_unsubscribe=False) + return 2 + return 1 + + + def update_email(self, email): + """Set the email address for the participant. + """ + if not getattr(self.get_email(email), 'verified', False): + raise EmailNotVerified(email) + username = self.username + with self.db.get_cursor() as c: + add_event(c, 'participant', dict(id=self.id, action='set', values=dict(primary_email=email))) + c.run(""" + UPDATE participants + SET email_address=%(email)s + WHERE username=%(username)s + """, locals()) + self.set_attributes(email_address=email) + + + def verify_email(self, email, nonce): + if '' in (email, nonce): + return emails.VERIFICATION_MISSING + r = self.get_email(email) + if r is None: + return emails.VERIFICATION_FAILED + if r.verified: + assert r.nonce is None # and therefore, order of conditions matters + return emails.VERIFICATION_REDUNDANT + if not constant_time_compare(r.nonce, nonce): + return emails.VERIFICATION_FAILED + if (utcnow() - r.verification_start) > EMAIL_HASH_TIMEOUT: + return emails.VERIFICATION_EXPIRED + try: + self.db.run(""" + UPDATE emails + SET verified=true, verification_end=now(), nonce=NULL + WHERE participant_id=%s + AND address=%s + AND verified IS NULL + """, (self.id, email)) + except IntegrityError: + return emails.VERIFICATION_STYMIED + + if not self.email_address: + self.update_email(email) + return emails.VERIFICATION_SUCCEEDED + + + def get_email(self, email): + """Return a record for a single email address on file for this participant. + """ + return self.db.one(""" + SELECT * + FROM emails + WHERE participant_id=%s + AND address=%s + """, (self.id, email)) + + + def get_emails(self): + """Return a list of all email addresses on file for this participant. + """ + return self.db.all(""" + SELECT * + FROM emails + WHERE participant_id=%s + ORDER BY id + """, (self.id,)) + + + def get_verified_email_addresses(self): + """Return a list of verified email addresses on file for this participant. + """ + return [email.address for email in self.get_emails() if email.verified] + + + def remove_email(self, address): + """Remove the given email address from the participant's account. + Raises ``CannotRemovePrimaryEmail`` if the address is primary. It's a + noop if the email address is not on file. + """ + if address == self.email_address: + raise CannotRemovePrimaryEmail() + with self.db.get_cursor() as c: + add_event(c, 'participant', dict(id=self.id, action='remove', values=dict(email=address))) + c.run("DELETE FROM emails WHERE participant_id=%s AND address=%s", + (self.id, address)) + + + def queue_email(self, spt_name, **context): + """Given the name of a template under ``emails/`` and context in + kwargs, queue a message to be sent. + """ + self.db.run(""" + INSERT INTO email_queue + (participant, spt_name, context) + VALUES (%s, %s, %s) + """, (self.id, spt_name, pickle.dumps(context))) + + + @classmethod + def dequeue_emails(cls): + """Load messages queued for sending, and send them. + """ + fetch_messages = lambda: cls.db.all(""" + SELECT * + FROM email_queue + ORDER BY id ASC + LIMIT 60 + """) + nsent = 0 + while True: + messages = fetch_messages() + if not messages: + break + for msg in messages: + p = cls.from_id(msg.participant) + r = p.send_email(msg.spt_name, **pickle.loads(msg.context)) + cls.db.run("DELETE FROM email_queue WHERE id = %s", (msg.id,)) + if r == 1: + sleep(1) + nsent += r + return nsent + + + def send_email(self, spt_name, **context): + """Given the name of an email template and context in kwargs, send an + email using the configured mailer. + """ + context['participant'] = self + context['username'] = self.username + context['button_style'] = ( + "color: #fff; text-decoration:none; display:inline-block; " + "padding: 0 15px; background: #396; white-space: nowrap; " + "font: normal 14px/40px Arial, sans-serif; border-radius: 3px" + ) + context.setdefault('include_unsubscribe', True) + email = context.setdefault('email', self.email_address) + if not email: + return 0 # Not Sent + langs = i18n.parse_accept_lang(self.email_lang or 'en') + locale = i18n.match_lang(langs) + i18n.add_helpers_to_context(self._tell_sentry, context, locale) + context['escape'] = lambda s: s + context_html = dict(context) + i18n.add_helpers_to_context(self._tell_sentry, context_html, locale) + context_html['escape'] = htmlescape + spt = self._emails[spt_name] + base_spt = self._emails['base'] + def render(t, context): + b = base_spt[t].render(context).strip() + return b.replace('$body', spt[t].render(context).strip()) + + message = {} + message['Source'] = 'Gratipay Support ' + message['Destination'] = {} + message['Destination']['ToAddresses'] = ["%s <%s>" % (self.username, email)] # "Name " + message['Message'] = {} + message['Message']['Subject'] = {} + message['Message']['Subject']['Data'] = spt['subject'].render(context).strip() + message['Message']['Body'] = { + 'Text': { + 'Data': render('text/plain', context) + }, + 'Html': { + 'Data': render('text/html', context_html) + } + } + + self._mailer.send_email(**message) + return 1 # Sent + + + def set_email_lang(self, accept_lang): + """Given a language identifier, set it for the participant as their + preferred language in which to receive email. + """ + if not accept_lang: + return + self.db.run("UPDATE participants SET email_lang=%s WHERE id=%s", + (accept_lang, self.id)) + self.set_attributes(email_lang=accept_lang) diff --git a/gratipay/testing/emails.py b/gratipay/testing/emails.py index 6e4c4d3927..b918009746 100644 --- a/gratipay/testing/emails.py +++ b/gratipay/testing/emails.py @@ -11,7 +11,7 @@ def setUp(self): self.mailer_patcher = mock.patch.object(Participant._mailer, 'send_email') self.mailer = self.mailer_patcher.start() self.addCleanup(self.mailer_patcher.stop) - sleep_patcher = mock.patch('gratipay.models.participant.sleep') + sleep_patcher = mock.patch('gratipay.models.participant.email.sleep') sleep_patcher.start() self.addCleanup(sleep_patcher.stop) diff --git a/tests/py/test_email.py b/tests/py/test_email.py index b64971d303..291f1f4c2b 100644 --- a/tests/py/test_email.py +++ b/tests/py/test_email.py @@ -13,7 +13,7 @@ from gratipay.utils.emails import queue_branch_email as _queue_branch_email -class TestEmail(EmailHarness): +class AliceAndResend(EmailHarness): def setUp(self): EmailHarness.setUp(self) @@ -24,6 +24,9 @@ def setUp(self): def tearDown(self): self.client.website.env.resend_verification_threshold = self._old_threshold + +class TestEndpoints(AliceAndResend): + def hit_email_spt(self, action, address, user='alice', should_fail=False): f = self.client.PxST if should_fail else self.client.POST data = {'action': action, 'address': address} @@ -169,6 +172,38 @@ def test_nonce_is_reused_when_resending_email(self): nonce2 = self.alice.get_email('alice@example.com').nonce assert nonce1 == nonce2 + def test_emails_page_shows_emails(self): + self.verify_and_change_email('alice@example.com', 'alice@example.net') + body = self.client.GET("/~alice/emails/", auth_as="alice").body + assert 'alice@example.com' in body + assert 'alice@example.net' in body + + def test_set_primary(self): + self.verify_and_change_email('alice@example.com', 'alice@example.net') + self.verify_and_change_email('alice@example.net', 'alice@example.org') + self.hit_email_spt('set-primary', 'alice@example.com') + + def test_cannot_set_primary_to_unverified(self): + with self.assertRaises(EmailNotVerified): + self.hit_email_spt('set-primary', 'alice@example.com') + + def test_remove_email(self): + # Can remove unverified + self.hit_email_spt('add-email', 'alice@example.com') + self.hit_email_spt('remove', 'alice@example.com') + + # Can remove verified + self.verify_and_change_email('alice@example.com', 'alice@example.net') + self.verify_and_change_email('alice@example.net', 'alice@example.org') + self.hit_email_spt('remove', 'alice@example.net') + + # Cannot remove primary + with self.assertRaises(CannotRemovePrimaryEmail): + self.hit_email_spt('remove', 'alice@example.com') + + +class TestFunctions(AliceAndResend): + def test_cannot_update_email_to_already_verified(self): bob = self.make_participant('bob', claimed_time='now') self.alice.add_email('alice@gratipay.com') @@ -209,35 +244,6 @@ def test_can_resend_verification_after_a_while(self): time.sleep(0.15) self.alice.add_email('alice@gratipay.coop', '0.1 seconds') - def test_emails_page_shows_emails(self): - self.verify_and_change_email('alice@example.com', 'alice@example.net') - body = self.client.GET("/~alice/emails/", auth_as="alice").body - assert 'alice@example.com' in body - assert 'alice@example.net' in body - - def test_set_primary(self): - self.verify_and_change_email('alice@example.com', 'alice@example.net') - self.verify_and_change_email('alice@example.net', 'alice@example.org') - self.hit_email_spt('set-primary', 'alice@example.com') - - def test_cannot_set_primary_to_unverified(self): - with self.assertRaises(EmailNotVerified): - self.hit_email_spt('set-primary', 'alice@example.com') - - def test_remove_email(self): - # Can remove unverified - self.hit_email_spt('add-email', 'alice@example.com') - self.hit_email_spt('remove', 'alice@example.com') - - # Can remove verified - self.verify_and_change_email('alice@example.com', 'alice@example.net') - self.verify_and_change_email('alice@example.net', 'alice@example.org') - self.hit_email_spt('remove', 'alice@example.net') - - # Cannot remove primary - with self.assertRaises(CannotRemovePrimaryEmail): - self.hit_email_spt('remove', 'alice@example.com') - def test_html_escaping(self): self.alice.add_email("foo'bar@example.com") last_email = self.get_last_email()