Merge pull request #153 from freedomofpress/https-onions

Use HTTPS for instances that support it
freedomofpress · Jan 24, 2024 · 7daa008 · 7daa008
2 parents 5a5524c + 4385bb2
commit 7daa008
Show file tree

Hide file tree

Showing 8 changed files with 32 additions and 20 deletions.
diff --git a/default.rulesets.1706124696.gz b/default.rulesets.1706124696.gz
diff --git a/index.html b/index.html
@@ -1,5 +1,5 @@
 <html>
-    <a href='rulesets-signature.1705008530.sha256'>rulesets-signature.1705008530.sha256</a><br>
+    <a href='rulesets-signature.1706124696.sha256'>rulesets-signature.1706124696.sha256</a><br>
     <a href='latest-rulesets-timestamp'>latest-rulesets-timestamp</a><br>
-    <a href='default.rulesets.1705008530.gz'>default.rulesets.1705008530.gz</a><br>
+    <a href='default.rulesets.1706124696.gz'>default.rulesets.1706124696.gz</a><br>
 </html>
diff --git a/latest-rulesets-timestamp b/latest-rulesets-timestamp
@@ -1 +1 @@
-1705008530
+1706124696
diff --git a/onboarded.txt b/onboarded.txt
@@ -1,18 +1,18 @@
-primary_domain,sd_rewrite_rule
+primary_domain,sd_rewrite_rule,is_https
 theintercept.com,theintercept.securedrop.tor.onion
 www.theguardian.com,theguardian.securedrop.tor.onion
 www.abc.net.au,abc.au.securedrop.tor.onion
 www.hmdc.harvard.edu,iqss.harvard.securedrop.tor.onion
 espenandersen.no,espena.securedrop.tor.onion
-www.nytimes.com,nytimes.securedrop.tor.onion
+www.nytimes.com,nytimes.securedrop.tor.onion,yes
 techcrunch.com,techcrunch.securedrop.tor.onion
 stefaniamaurizi.it,maurizi.securedrop.tor.onion
 www.forbes.com,forbes.securedrop.tor.onion
 www.thestar.com,torontostar.securedrop.tor.onion
 tcfmailvault.info,bartongellman.securedrop.tor.onion
 www.apache.be,apache.securedrop.tor.onion
 www.2600.com,2600.securedrop.tor.onion
-www.washingtonpost.com,washingtonpost.securedrop.tor.onion
+www.washingtonpost.com,washingtonpost.securedrop.tor.onion,yes
 www.bloomberg.com,bloomberg.securedrop.tor.onion
 www.aftonbladet.se,aftonbladet.securedrop.tor.onion
 www.aftenposten.no,aftenposten.securedrop.tor.onion

diff --git a/rulesets-signature.1706124696.sha256 b/rulesets-signature.1706124696.sha256
diff --git a/rulesets/new-york-times-securedrop-ruleset.xml b/rulesets/new-york-times-securedrop-ruleset.xml
@@ -1,5 +1,5 @@
 <ruleset name="New York Times">
 	<target host="nytimes.securedrop.tor.onion" />
 	<rule from="^http[s]?://nytimes.securedrop.tor.onion"
-    to="http://ej3kv4ebuugcmuwxctx5ic7zxh73rnxt42soi3tdneu2c2em55thufqd.onion" />
+    to="https://ej3kv4ebuugcmuwxctx5ic7zxh73rnxt42soi3tdneu2c2em55thufqd.onion" />
 </ruleset>
diff --git a/rulesets/washington-post-securedrop-ruleset.xml b/rulesets/washington-post-securedrop-ruleset.xml
@@ -1,5 +1,5 @@
 <ruleset name="The Washington Post">
 	<target host="washingtonpost.securedrop.tor.onion" />
 	<rule from="^http[s]?://washingtonpost.securedrop.tor.onion"
-    to="http://vfnmxpa6fo4jdpyq3yneqhglluweax2uclvxkytfpmpkp5rsl75ir5qd.onion" />
+    to="https://vfnmxpa6fo4jdpyq3yneqhglluweax2uclvxkytfpmpkp5rsl75ir5qd.onion" />
 </ruleset>
diff --git a/sddir.py b/sddir.py
@@ -3,16 +3,16 @@
 import os
 import re
 import requests
+import textwrap
 import urllib
-
+from urllib.parse import urlparse
 from typing import Dict
 
 # Configure logging output
 logfmt = "%(asctime)s %(levelname)-8s %(message)s"
 logging.basicConfig(format=logfmt, level=logging.DEBUG, datefmt="%Y-%m-%d %H:%M:%S")
 
 SECUREDROP_ONION_PSEUDO_TLD = ".securedrop.tor.onion"
-DEFAULT_ONION_PROTOCOL = "http://"  # We don't store protocol in the directory
 RULESET_DIR = "rulesets"
 
 # tcfmailvault.info = unlisted SecureDrop; all others: extended outage
@@ -47,9 +47,6 @@ def get_securedrop_directory() -> Dict:
         directory_entry["securedrop_redirect_url"] = (
             directory_entry["base_url"].rsplit(".", 1)[0] + SECUREDROP_ONION_PSEUDO_TLD
         )
-        directory_entry["onion_addr_with_protocol"] = (
-            DEFAULT_ONION_PROTOCOL + directory_entry["onion_address"]
-        )
         directory_entry["slug"] = remove_umlaut(directory_entry["slug"])
         directory_entry["title"] = remove_umlaut(directory_entry["title"])
 
@@ -58,19 +55,31 @@ def get_securedrop_directory() -> Dict:
     return directory_entry_map
 
 
-def write_custom_ruleset(onboarded_org: str, sd_rewrite_rule: str, directory_entries: Dict) -> None:
+def write_custom_ruleset(
+    onboarded_org: str, sd_rewrite_rule: str, is_https: bool, directory_entries: Dict
+) -> None:
     try:
         directory_entry = directory_entries[onboarded_org]
     except KeyError:
         logging.error(f"Failed to find '{onboarded_org}', org names are:")
         logging.error(directory_entries.keys())
         raise
 
-    ruleset = """<ruleset name="{org_name}">\n\t<target host="{securedrop_redirect_url}" />\n\t<rule from="^http[s]?://{securedrop_redirect_url}"
-    to="{onion_addr_with_protocol}" />\n</ruleset>\n""".format(
-        org_name=directory_entry["title"],
-        securedrop_redirect_url=sd_rewrite_rule,
-        onion_addr_with_protocol=directory_entry["onion_addr_with_protocol"],
+    secure = "s" if is_https else ""
+    onion_addr = f"http{secure}://{directory_entry['onion_address']}"
+
+    ruleset = textwrap.dedent(
+        """\
+    <ruleset name="{org_name}">
+    \t<target host="{securedrop_redirect_url}" />
+    \t<rule from="^http[s]?://{securedrop_redirect_url}"
+        to="{onion_addr}" />
+    </ruleset>
+    """.format(
+            org_name=directory_entry["title"],
+            securedrop_redirect_url=sd_rewrite_rule,
+            onion_addr=onion_addr,
+        )
     )
 
     RULESET_OUTPUT = "securedrop-ruleset.xml"
@@ -92,6 +101,9 @@ def write_custom_ruleset(onboarded_org: str, sd_rewrite_rule: str, directory_ent
             if row["primary_domain"] in EXEMPTIONS:
                 logging.warning(f"Skipping exempted domain: {row['primary_domain']}")
                 continue
-            write_custom_ruleset(row["primary_domain"], row["sd_rewrite_rule"], directory_entries)
+            is_https = row["is_https"] == "yes"
+            write_custom_ruleset(
+                row["primary_domain"], row["sd_rewrite_rule"], is_https, directory_entries
+            )
 
     logging.info("✔️ Custom rulesets written to directory: ./{}".format(RULESET_DIR))