Treat messages and metadata as attachments

README.md

@@ -19,7 +19,8 @@ In `commons.py` there are the following configuration values which are global fo
 |---|---|---|---|
 | `SERVER` | `127.0.0.1:5000` | source, journalist | The URL the Flask server listens on; used by both the journalist and the source clients. |
 | `DIR` | `keys/` | server, source, journalist | The folder where everybody will load the keys from. There is no separation for demo simplicity of course in an actual implementation, everybody will only have their keys and the required public one to ensure the trust chain. |
-| `UPLOADS` | `files/` | server | The folder where the Flask server will store uploaded files
+| `UPLOADS` | `files/` | server | The folder where the Flask server will store uploaded files. |
+| `DOWNLOADS` | `downloads/` | journalist | The folder where submission attachments will be saved. |
 | `JOURNALISTS` | `10` | server, source |  How many journalists do we create and enroll. In general, this is realistic, in current SecureDrop usage it is way less. For demo purposes everybody knows it, in a real scenario it would not be needed. |
 | `ONETIMEKEYS` | `30` | journalist | How many ephemeral keys each journalist create, sign and uploads when required. |
 | `CURVE` | `NIST384p` | server, source, journalist | The curve for all elliptic curve operations. It must be imported first from the python-ecdsa library. Ed25519 and Ed448, although supported by the lib, are not fully implemented. |

commons.py

@@ -18,6 +18,8 @@
 DIR = "keys/"
 # Where the flask server will store uploaded files
 UPLOADS = "files/"
+# Folders where journalists download attachments to submissions
+DOWNLOADS = "downloads/"
 # How many journalists do we create and enroll. In general, this is realistic, in current
 # securedrop usage it is way less
 JOURNALISTS = 10
@@ -229,7 +231,7 @@ def fetch_messages_content(messages_id):
         return messages_list
 
 
-def decrypt_message_ciphertext(private_key, message_public_key, message_ciphertext):
+def decrypt_message_asymmetric(private_key, message_public_key, message_ciphertext):
     ecdh = ECDH(curve=CURVE)
     ecdh.load_private_key(private_key)
     ecdh.load_received_public_key_bytes(b64decode(message_public_key))
@@ -242,6 +244,21 @@ def decrypt_message_ciphertext(private_key, message_public_key, message_cipherte
         return False
 
 
+def upload_message(content):
+    assert (len(content) < CHUNK)
+    key = token_bytes(32)
+    box = nacl.secret.SecretBox(key)
+    ciphertext = box.encrypt(content.ljust(CHUNK).encode('ascii'))
+    upload_response = send_file(ciphertext)
+    return upload_response['file_id'], key.hex()
+
+
+def decrypt_message_symmetric(ciphertext, key):
+    box = nacl.secret.SecretBox(key)
+    plaintext = json.loads(box.decrypt(ciphertext).decode('ascii'))
+    return plaintext
+
+
 def upload_attachment(filename):
     try:
         size = stat(filename).st_size

journalist.py

@@ -2,17 +2,17 @@
 import json
 from base64 import b64encode
 from datetime import datetime
+from hashlib import sha3_256
 from os import listdir, mkdir, path
 from time import time
 
 import nacl.secret
 import requests
 from ecdsa import SigningKey
-from hashlib import sha3_256
 
 import commons
-import pki
 import journalist_db
+import pki
 
 
 def add_ephemeral_keys(journalist_key, journalist_id, journalist_uid):
@@ -48,7 +48,7 @@ def load_ephemeral_keys(journalist_key, journalist_id, journalist_uid):
 # This is inefficient, but on an actual implementation we would discard already used keys
 def decrypt_message(ephemeral_keys, message):
     for ephemeral_key in ephemeral_keys:
-        message_plaintext = commons.decrypt_message_ciphertext(
+        message_plaintext = commons.decrypt_message_asymmetric(
             ephemeral_key, message["message_public_key"],
             message["message_ciphertext"])
         if message_plaintext:
@@ -72,8 +72,10 @@ def journalist_reply(message, reply, journalist_uid):
                     "group_members": [],
                     "timestamp": int(time())}
 
-    message_ciphertext = b64encode(
-        box.encrypt((json.dumps(message_dict)).ljust(1024).encode('ascii'))
+    file_id, key = commons.upload_message(json.dumps(message_dict))
+
+    message_ciphertext = b64encode(box.encrypt(
+        (json.dumps({"file_id": file_id, "key": key})).encode('ascii'))
     ).decode("ascii")
 
     # Send the message to the server API using the generic /send endpoint
@@ -116,14 +118,20 @@ def main(args):
         message_id = args.id
         message = commons.get_message(message_id)
         ephemeral_keys = load_ephemeral_keys(journalist_key, journalist_id, journalist_uid)
+        # Get the encrypted file_id and decryption key of the message
         message_plaintext = decrypt_message(ephemeral_keys, message)
 
+        # Fetch and decrypt the actual message, that was stored as an attachment
+        key = message_plaintext['key']
+        encrypted_message_content = commons.get_file(message_plaintext['file_id'])
+        message_plaintext = commons.decrypt_message_symmetric(encrypted_message_content, bytes.fromhex(key))
+
         if message_plaintext:
             # Create a download folder if we have attachments
             if (message_plaintext["attachments"] and
                len(message_plaintext["attachments"]) > 0):
                 try:
-                    mkdir('downloads/')
+                    mkdir(commons.DOWNLOADS)
                 except Exception:
                     pass
             else:
@@ -139,7 +147,7 @@ def main(args):
                 print(f"\tAttachment: name={attachment['name']};size={attachment['size']};parts_count={attachment['parts_count']}")
                 attachment_name = path.basename(attachment['name'])
                 attachment_size = attachment['size']
-                with open(f"downloads/{int(time())}_{attachment_name}", "wb") as f:
+                with open(f"{commons.DOWNLOADS}{int(time())}_{attachment_name}", "wb") as f:
                     part_number = 0
                     written_size = 0
                     while written_size < attachment_size:
@@ -166,7 +174,11 @@ def main(args):
         message_id = args.id
         message = commons.get_message(message_id)
         ephemeral_keys = load_ephemeral_keys(journalist_key, journalist_id, journalist_uid)
-        message_plaintext = decrypt_message(ephemeral_keys, message)
+        envelope_plaintext = decrypt_message(ephemeral_keys, message)
+        message_ciphertext = commons.get_file(envelope_plaintext['file_id'])
+        message_symmetric_key = bytes.fromhex(envelope_plaintext['key'])
+        message_plaintext = commons.decrypt_message_symmetric(message_ciphertext,
+                                                              message_symmetric_key)
         journalist_reply(message_plaintext, args.message, journalist_uid)
 
     elif args.action == "delete":

journalist_db.py

@@ -1,13 +1,14 @@
 import os
 import sqlite3
 
+
 class JournalistDatabase():
 
     def __init__(self, path):
         self.path = path
         self.is_valid = os.path.isfile(self.path)
         self.con = sqlite3.connect(self.path)
-        if self.is_valid == False:
+        if self.is_valid is False:
             try:
                 self.create()
             except sqlite3.OperationalError:

source.py

@@ -44,28 +44,29 @@ def send_submission(intermediate_verifying_key, passphrase, message, attachments
     challenge_key = derive_key(passphrase, "challenge_key-")
     source_challenge_public_key = b64encode(challenge_key.verifying_key.to_string()).decode("ascii")
 
+    # Same as on the journalist side: this structure is built by the clients
+    # and thus potentially "untrusted"
+    message_dict = {"message": message,
+                    # do we want to sign messages? how do we attest source authoriship?
+                    "source_challenge_public_key": source_challenge_public_key,
+                    "source_encryption_public_key": source_encryption_public_key,
+                    # we could list the journalists involved in the conversation here
+                    # if the source choose not to pick everybody
+                    "group_members": [],
+                    "timestamp": int(time()),
+                    # we can add attachmenet pieces/id here
+                    "attachments": attachments}
+
+    file_id, key = commons.upload_message(json.dumps(message_dict))
+
     # For every receiver (journalists), create a message
     for ephemeral_key_dict in ephemeral_keys:
         # This function builds the per-message keys and returns a nacl encrypting box
         message_public_key, message_challenge, box = commons.build_message(ephemeral_key_dict["journalist_chal_key"],
                                                                            ephemeral_key_dict["ephemeral_key"])
 
-        # Same as on the journalist side: this structure is built by the clients
-        # and thus potentially "untrusted"
-        message_dict = {"message": message,
-                        # do we want to sign messages? how do we attest source authoriship?
-                        "source_challenge_public_key": source_challenge_public_key,
-                        "source_encryption_public_key": source_encryption_public_key,
-                        "receiver": ephemeral_key_dict["journalist_uid"],
-                        # we could list the journalists involved in the conversation here
-                        # if the source choose not to pick everybody
-                        "group_members": [],
-                        "timestamp": int(time()),
-                        # we can add attachmenet pieces/id here
-                        "attachments": attachments}
-
         message_ciphertext = b64encode(box.encrypt(
-            (json.dumps(message_dict)).ljust(1024).encode('ascii'))
+            (json.dumps({"file_id": file_id, "key": key})).encode('ascii'))
         ).decode("ascii")
 
         # Send the message to the server API using the generic /send endpoint
@@ -127,13 +128,17 @@ def main(args):
         source_key = derive_key(passphrase, "source_key-")
         message_id = args.id
         message = commons.get_message(message_id)
-        message_plaintext = commons.decrypt_message_ciphertext(source_key,
+        message_plaintext = commons.decrypt_message_asymmetric(source_key,
                                                                message["message_public_key"],
                                                                message["message_ciphertext"])
 
         if message_plaintext:
             print(f"[+] Successfully decrypted message {message_id}")
+            print(f"[+] file_id: {message_plaintext['file_id']}, key: {message_plaintext['key']}")
             print()
+            key = message_plaintext['key']
+            encrypted_message_content = commons.get_file(message_plaintext['file_id'])
+            message_plaintext = commons.decrypt_message_symmetric(encrypted_message_content, bytes.fromhex(key))
             print(f"\tID: {message_id}")
             print(f"\tFrom: {message_plaintext['sender']}")
             print(f"\tDate: {datetime.fromtimestamp(message_plaintext['timestamp'])}")