From c64b3ba93ef933cc335f2de993cc8a11fee23fc2 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Sun, 8 May 2016 20:38:20 -0400 Subject: [PATCH] add rekeying --- bin/rekey.py | 8 ++- .../models/participant/mixins/identity.py | 51 +++++++++++++++++++ gratipay/wireup.py | 3 +- sql/branch.sql | 11 ++-- tests/py/test_close.py | 4 +- tests/py/test_participant_identities.py | 30 ++++++++++- 6 files changed, 96 insertions(+), 11 deletions(-) diff --git a/bin/rekey.py b/bin/rekey.py index 61ceb0b4cd..a1c20dc1de 100755 --- a/bin/rekey.py +++ b/bin/rekey.py @@ -1,10 +1,14 @@ #!/usr/bin/env python2 +"""See gratipay.models.participant.mixins.identity.rekey for documentation. +""" from __future__ import absolute_import, division, print_function, unicode_literals from gratipay import wireup +from gratipay.models.participant.mixins import identity as participant_identities env = wireup.env() db = wireup.db(env) -wireup.crypto(env) +packer = wireup.crypto(env) -print("{} record(s) rekeyed.".format(0)) # stubbed until we have something to rekey +n = participant_identities.rekey(db, packer) +print("Rekeyed {} participant identity record(s).".format(n)) diff --git a/gratipay/models/participant/mixins/identity.py b/gratipay/models/participant/mixins/identity.py index a90b9bd449..7fd0e7a2cf 100644 --- a/gratipay/models/participant/mixins/identity.py +++ b/gratipay/models/participant/mixins/identity.py @@ -164,3 +164,54 @@ def list_identity_metadata(self): ORDER BY c.code """, (self.id,)) + + +# Rekeying +# ======== + +def rekey(db, packer): + """Rekey the encrypted participant identity information in our database. + + :param GratipayDB db: used to access the database + :param EncryptingPacker packer: used to decrypt and encrypt data + + This function features prominently in our procedure for rekeying our + encrypted data, as documented in the "`Keep Secrets`_" howto. It operates + by loading records from `participant_identities` that haven't been updated + in the present month, in batches of 100. It updates a timestamp atomically + with each rekeyed `info`, so it can be safely rerun in the face of network + failure, etc. + + .. _Keep Secrets: http://inside.gratipay.com/howto/keep-secrets + + """ + n = 0 + while 1: + m = _rekey_one_batch(db, packer) + if m == 0: + break + n += m + return n + + +def _rekey_one_batch(db, packer): + batch = db.all(""" + + SELECT id, info + FROM participant_identities + WHERE _info_last_keyed < date_trunc('month', now()) + ORDER BY _info_last_keyed ASC + LIMIT 100 + + """) + if not batch: + return 0 + + for rec in batch: + plaintext = packer.unpack(bytes(rec.info)) + new_token = packer.pack(plaintext) + db.run( "UPDATE participant_identities SET info=%s, _info_last_keyed=now() WHERE id=%s" + , (new_token, rec.id) + ) + + return len(batch) diff --git a/gratipay/wireup.py b/gratipay/wireup.py index b5df7bdc40..8aa1dd220e 100644 --- a/gratipay/wireup.py +++ b/gratipay/wireup.py @@ -64,7 +64,8 @@ def db(env): def crypto(env): keys = [k.encode('ASCII') for k in env.crypto_keys.split()] - Identity.encrypting_packer = EncryptingPacker(*keys) + out = Identity.encrypting_packer = EncryptingPacker(*keys) + return out def mail(env, project_root='.'): if env.aws_ses_access_key_id and env.aws_ses_secret_access_key and env.aws_ses_default_region: diff --git a/sql/branch.sql b/sql/branch.sql index 40930db990..d82835a355 100644 --- a/sql/branch.sql +++ b/sql/branch.sql @@ -1,9 +1,10 @@ CREATE TABLE participant_identities -( id bigserial primary key -, participant_id bigint NOT NULL REFERENCES participants(id) -, country_id bigint NOT NULL REFERENCES countries(id) -, schema_name text NOT NULL -, info bytea NOT NULL +( id bigserial primary key +, participant_id bigint NOT NULL REFERENCES participants(id) +, country_id bigint NOT NULL REFERENCES countries(id) +, schema_name text NOT NULL +, info bytea NOT NULL +, _info_last_keyed timestamptz NOT NULL DEFAULT now() , UNIQUE(participant_id, country_id) ); diff --git a/tests/py/test_close.py b/tests/py/test_close.py index fe9e98d681..88ca8a59cc 100644 --- a/tests/py/test_close.py +++ b/tests/py/test_close.py @@ -179,13 +179,13 @@ def test_cpi_clears_personal_identities(self): US = self.db.one("SELECT id FROM countries WHERE code='US'") alice.store_identity_info(US, 'nothing-enforced', {'name': 'Alice'}) assert len(alice.list_identity_metadata()) == 1 - assert len(self.db.all('SELECT * FROM participant_identities;')) == 1 + assert self.db.one('SELECT count(*) FROM participant_identities;') == 1 with self.db.get_cursor() as cursor: alice.clear_personal_information(cursor) assert len(alice.list_identity_metadata()) == 0 - assert len(self.db.all('SELECT * FROM participant_identities;')) == 0 + assert self.db.one('SELECT count(*) FROM participant_identities;') == 0 # uic = update_is_closed diff --git a/tests/py/test_participant_identities.py b/tests/py/test_participant_identities.py index ee5320d507..2e525df86b 100644 --- a/tests/py/test_participant_identities.py +++ b/tests/py/test_participant_identities.py @@ -1,10 +1,12 @@ from __future__ import absolute_import, division, print_function, unicode_literals +from cryptography.fernet import InvalidToken from gratipay.testing import Harness from gratipay.models.participant.mixins import identity, Identity -from gratipay.models.participant.mixins.identity import _validate_info +from gratipay.models.participant.mixins.identity import _validate_info, rekey from gratipay.models.participant.mixins.identity import ParticipantIdentityInfoInvalid from gratipay.models.participant.mixins.identity import ParticipantIdentitySchemaUnknown +from gratipay.security.crypto import EncryptingPacker, Fernet from psycopg2 import IntegrityError from pytest import raises @@ -149,3 +151,29 @@ def test_fine_fails_if_no_email(self): ).value assert error.pgcode == '23100' assert bruiser.list_identity_metadata() == [] + + + # rekey + + def rekey_setup(self): + self.crusher.store_identity_info(self.US, 'nothing-enforced', {'name': 'Crusher'}) + self.db.run("UPDATE participant_identities " + "SET _info_last_keyed=_info_last_keyed - '6 months'::interval") + old_key = str(self.client.website.env.crypto_keys) + return EncryptingPacker(Fernet.generate_key(), old_key) + + def test_rekey_rekeys(self): + assert rekey(self.db, self.rekey_setup()) == 1 + + def test_rekeying_causes_old_packer_to_fail(self): + rekey(self.db, self.rekey_setup()) + raises(InvalidToken, self.crusher.retrieve_identity_info, self.US) + + def test_rekeyed_data_is_accessible_with_new_key(self): + self.crusher.encrypting_packer = self.rekey_setup() + assert self.crusher.retrieve_identity_info(self.US) == {'name': 'Crusher'} + + def test_rekey_ignores_recently_keyed_records(self): + self.crusher.encrypting_packer = self.rekey_setup() + assert rekey(self.db, self.crusher.encrypting_packer) == 1 + assert rekey(self.db, self.crusher.encrypting_packer) == 0