add rekeying

bin/rekey.py

@@ -1,10 +1,14 @@
 #!/usr/bin/env python2
+"""See gratipay.models.participant.mixins.identity.rekey for documentation.
+"""
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from gratipay import wireup
+from gratipay.models.participant.mixins import identity as participant_identities
 
 env = wireup.env()
 db = wireup.db(env)
-wireup.crypto(env)
+packer = wireup.crypto(env)
 
-print("{} record(s) rekeyed.".format(0))  # stubbed until we have something to rekey
+n = participant_identities.rekey(db, packer)
+print("Rekeyed {} participant identity record(s).".format(n))

gratipay/models/participant/mixins/identity.py

@@ -164,3 +164,54 @@ def list_identity_metadata(self):
           ORDER BY c.code
 
         """, (self.id,))
+
+
+# Rekeying
+# ========
+
+def rekey(db, packer):
+    """Rekey the encrypted participant identity information in our database.
+
+    :param GratipayDB db: used to access the database
+    :param EncryptingPacker packer: used to decrypt and encrypt data
+
+    This function features prominently in our procedure for rekeying our
+    encrypted data, as documented in the "`Keep Secrets`_" howto. It operates
+    by loading records from `participant_identities` that haven't been updated
+    in the present month, in batches of 100. It updates a timestamp atomically
+    with each rekeyed `info`, so it can be safely rerun in the face of network
+    failure, etc.
+
+    .. _Keep Secrets: http://inside.gratipay.com/howto/keep-secrets
+
+    """
+    n = 0
+    while 1:
+        m = _rekey_one_batch(db, packer)
+        if m == 0:
+            break
+        n += m
+    return n
+
+
+def _rekey_one_batch(db, packer):
+    batch = db.all("""
+
+        SELECT id, info
+          FROM participant_identities
+         WHERE _info_last_keyed < date_trunc('month', now())
+      ORDER BY _info_last_keyed ASC
+         LIMIT 100
+
+    """)
+    if not batch:
+        return 0
+
+    for rec in batch:
+        plaintext = packer.unpack(bytes(rec.info))
+        new_token = packer.pack(plaintext)
+        db.run( "UPDATE participant_identities SET info=%s, _info_last_keyed=now() WHERE id=%s"
+              , (new_token, rec.id)
+               )
+
+    return len(batch)

gratipay/wireup.py

@@ -64,7 +64,8 @@ def db(env):
 
 def crypto(env):
     keys = [k.encode('ASCII') for k in env.crypto_keys.split()]
-    Identity.encrypting_packer = EncryptingPacker(*keys)
+    out = Identity.encrypting_packer = EncryptingPacker(*keys)
+    return out
 
 def mail(env, project_root='.'):
     if env.aws_ses_access_key_id and env.aws_ses_secret_access_key and env.aws_ses_default_region:

sql/branch.sql

@@ -1,9 +1,10 @@
 CREATE TABLE participant_identities
-( id                bigserial   primary key
-, participant_id    bigint      NOT NULL REFERENCES participants(id)
-, country_id        bigint      NOT NULL REFERENCES countries(id)
-, schema_name       text        NOT NULL
-, info              bytea       NOT NULL
+( id                bigserial       primary key
+, participant_id    bigint          NOT NULL REFERENCES participants(id)
+, country_id        bigint          NOT NULL REFERENCES countries(id)
+, schema_name       text            NOT NULL
+, info              bytea           NOT NULL
+, _info_last_keyed  timestamptz     NOT NULL DEFAULT now()
 , UNIQUE(participant_id, country_id)
  );
 

tests/py/test_close.py

@@ -179,13 +179,13 @@ def test_cpi_clears_personal_identities(self):
         USA = self.db.one("SELECT id FROM countries WHERE code3='USA'")
         alice.store_identity_info(USA, 'nothing-enforced', {'name': 'Alice'})
         assert len(alice.list_identity_metadata()) == 1
-        assert len(self.db.all('SELECT * FROM participant_identities;')) == 1
+        assert self.db.one('SELECT count(*) FROM participant_identities;') == 1
 
         with self.db.get_cursor() as cursor:
             alice.clear_personal_information(cursor)
 
         assert len(alice.list_identity_metadata()) == 0
-        assert len(self.db.all('SELECT * FROM participant_identities;')) == 0
+        assert self.db.one('SELECT count(*) FROM participant_identities;') == 0
 
 
     # uic = update_is_closed

tests/py/test_participant_identities.py

@@ -1,10 +1,12 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+from cryptography.fernet import InvalidToken
 from gratipay.testing import Harness
 from gratipay.models.participant.mixins import identity, Identity
-from gratipay.models.participant.mixins.identity import _validate_info
+from gratipay.models.participant.mixins.identity import _validate_info, rekey
 from gratipay.models.participant.mixins.identity import ParticipantIdentityInfoInvalid
 from gratipay.models.participant.mixins.identity import ParticipantIdentitySchemaUnknown
+from gratipay.security.crypto import EncryptingPacker, Fernet
 from psycopg2 import IntegrityError
 from pytest import raises
 
@@ -149,3 +151,29 @@ def test_fine_fails_if_no_email(self):
                        ).value
         assert error.pgcode == '23100'
         assert bruiser.list_identity_metadata() == []
+
+
+    # rekey
+
+    def rekey_setup(self):
+        self.crusher.store_identity_info(self.USA, 'nothing-enforced', {'name': 'Crusher'})
+        self.db.run("UPDATE participant_identities "
+                    "SET _info_last_keyed=_info_last_keyed - '6 months'::interval")
+        old_key = str(self.client.website.env.crypto_keys)
+        return EncryptingPacker(Fernet.generate_key(), old_key)
+
+    def test_rekey_rekeys(self):
+        assert rekey(self.db, self.rekey_setup()) == 1
+
+    def test_rekeying_causes_old_packer_to_fail(self):
+        rekey(self.db, self.rekey_setup())
+        raises(InvalidToken, self.crusher.retrieve_identity_info, self.USA)
+
+    def test_rekeyed_data_is_accessible_with_new_key(self):
+        self.crusher.encrypting_packer = self.rekey_setup()
+        assert self.crusher.retrieve_identity_info(self.USA) == {'name': 'Crusher'}
+
+    def test_rekey_ignores_recently_keyed_records(self):
+        self.crusher.encrypting_packer = self.rekey_setup()
+        assert rekey(self.db, self.crusher.encrypting_packer) == 1
+        assert rekey(self.db, self.crusher.encrypting_packer) == 0