From d4e9256ae3674a6a3018f0cebfd504f799d23638 Mon Sep 17 00:00:00 2001 From: Martin Delille Date: Sat, 16 Mar 2019 20:58:15 +0100 Subject: [PATCH 1/9] protect community from confusable homoglyphs --- liberapay/models/community.py | 21 +++++++++++++++++++++ tests/py/test_communities.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/liberapay/models/community.py b/liberapay/models/community.py index e7b8cea56b..fb8f13b0c0 100644 --- a/liberapay/models/community.py +++ b/liberapay/models/community.py @@ -3,6 +3,7 @@ from postgres.orm import Model from psycopg2 import IntegrityError +from confusable_homoglyphs import confusables from liberapay.exceptions import CommunityAlreadyExists, InvalidCommunityName @@ -38,8 +39,17 @@ def create(cls, name, creator_id, lang='mul'): name = unicodedata.normalize('NFKC', name) if name_re.match(name) is None: raise InvalidCommunityName(name) + try: with cls.db.get_cursor() as cursor: + all_names = cursor.all(""" + SELECT name + FROM communities + """) + for existing_name in all_names: + if cls._unconfusable(name) == cls._unconfusable(existing_name): + raise CommunityAlreadyExists + p_id = cursor.one(""" INSERT INTO participants (kind, status, join_time) @@ -91,3 +101,14 @@ def check_membership_status(self, participant): @property def nsubscribers(self): return self.participant.nsubscribers + + @staticmethod + def _unconfusable(name): + unconfusable_name = '' + for c in name: + confusable = confusables.is_confusable(c, preferred_aliases=['COMMON', 'LATIN']) + if confusable: + # if the character is confusable we replace it with the first prefered alias + c = confusable[0]['homoglyphs'][0]['c'] + unconfusable_name += c + return unconfusable_name diff --git a/tests/py/test_communities.py b/tests/py/test_communities.py index 22e197b809..69929723ce 100644 --- a/tests/py/test_communities.py +++ b/tests/py/test_communities.py @@ -1,6 +1,9 @@ import json -from liberapay.exceptions import AuthRequired +from liberapay.exceptions import ( + AuthRequired, + CommunityAlreadyExists, +) from liberapay.models.community import Community from liberapay.testing import Harness @@ -101,6 +104,30 @@ def test_join_and_leave(self): communities = self.bob.get_communities() assert len(communities) == 0 + def test_create_community_already_taken(self): + with self.assertRaises(CommunityAlreadyExists): + Community.create('test', self.alice.id) + + def test_create_community_already_taken_is_case_insensitive(self): + with self.assertRaises(CommunityAlreadyExists): + Community.create('TeSt', self.alice.id) + + def test_unconfusable(self): + self.assertEqual('user2', Community._unconfusable('user2')) + self.assertEqual('alice', Community._unconfusable('alice')) + latin_string = 'AlaskaJazz' + mixed_string = 'ΑlaskaJazz' + self.assertNotEqual(latin_string, mixed_string) + self.assertEqual(latin_string, Community._unconfusable(mixed_string)) + + def test_create_community_already_taken_with_confusable_homoglyphs(self): + latin_string = 'AlaskaJazz' + mixed_string = 'ΑlaskaJazz' + + Community.create(latin_string, self.bob.id) + with self.assertRaises(CommunityAlreadyExists): + Community.create(mixed_string, self.alice.id) + class TestCommunityEdit(Harness): From 6f100de6a6a0d6d24a9907a4710811542dda4cbb Mon Sep 17 00:00:00 2001 From: Martin Delille Date: Sat, 16 Mar 2019 21:20:19 +0100 Subject: [PATCH 2/9] adding confusable_homoglyphs to requirements_base.txt --- requirements_base.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements_base.txt b/requirements_base.txt index b4e1847e87..e924826d9d 100644 --- a/requirements_base.txt +++ b/requirements_base.txt @@ -243,3 +243,6 @@ cryptography==2.4.2 \ asn1crypto==0.24.0 \ --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \ --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 +confusable_homoglyphs==3.2.0 \ + --hash=sha256:3b4a0d9fa510669498820c91a0bfc0c327568cecec90648cf3819d4a6fc6a751 \ + --hash=sha256:e3ce611028d882b74a5faa69e3cbb5bd4dcd9f69936da6e73d33eda42c917944 From b4d4c055c2484f73268dade93cfcec5c95569191 Mon Sep 17 00:00:00 2001 From: Martin Delille Date: Sat, 16 Mar 2019 21:28:44 +0100 Subject: [PATCH 3/9] fix flake8 test --- tests/py/test_communities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/py/test_communities.py b/tests/py/test_communities.py index 69929723ce..3119d5004a 100644 --- a/tests/py/test_communities.py +++ b/tests/py/test_communities.py @@ -1,8 +1,8 @@ import json from liberapay.exceptions import ( - AuthRequired, - CommunityAlreadyExists, + AuthRequired, + CommunityAlreadyExists, ) from liberapay.models.community import Community from liberapay.testing import Harness From 4883243897b38cc3060e4ecd40ed4625f28b8c70 Mon Sep 17 00:00:00 2001 From: Martin Delille Date: Sat, 16 Mar 2019 22:15:58 +0100 Subject: [PATCH 4/9] update confusable_homoglyphs in make env step --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 70692bf796..c43839c603 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ env: requirements*.txt $(python) -m $(pip) install $(install_where) "virtualenv>=15.0.0" $(python) -m virtualenv --no-download ./env/ $(env_bin)/$(pip) install --require-hashes $$(for f in requirements_*.txt; do echo "-r $$f"; done) + $(env_bin)/python -c "from confusable_homoglyphs import cli; cli.update()" @touch env rehash-requirements: From 9b496bc88f8a3c404394ee8c1999e33a6f944a0b Mon Sep 17 00:00:00 2001 From: Martin Delille Date: Thu, 21 Mar 2019 15:33:54 +0100 Subject: [PATCH 5/9] move unconfusable_string to liberapay/utils/unconfusable.py --- liberapay/models/community.py | 16 +++------------- liberapay/utils/unconfusable.py | 11 +++++++++++ tests/py/test_communities.py | 8 -------- tests/py/test_utils.py | 13 ++++++++++++- 4 files changed, 26 insertions(+), 22 deletions(-) create mode 100644 liberapay/utils/unconfusable.py diff --git a/liberapay/models/community.py b/liberapay/models/community.py index fb8f13b0c0..aa4e30b121 100644 --- a/liberapay/models/community.py +++ b/liberapay/models/community.py @@ -3,9 +3,9 @@ from postgres.orm import Model from psycopg2 import IntegrityError -from confusable_homoglyphs import confusables from liberapay.exceptions import CommunityAlreadyExists, InvalidCommunityName +from liberapay.utils.unconfusable import unconfusable_string name_maxlength = 40 @@ -42,12 +42,13 @@ def create(cls, name, creator_id, lang='mul'): try: with cls.db.get_cursor() as cursor: + unconfusable_name = unconfusable_string(name) all_names = cursor.all(""" SELECT name FROM communities """) for existing_name in all_names: - if cls._unconfusable(name) == cls._unconfusable(existing_name): + if unconfusable_name == unconfusable_string(existing_name): raise CommunityAlreadyExists p_id = cursor.one(""" @@ -101,14 +102,3 @@ def check_membership_status(self, participant): @property def nsubscribers(self): return self.participant.nsubscribers - - @staticmethod - def _unconfusable(name): - unconfusable_name = '' - for c in name: - confusable = confusables.is_confusable(c, preferred_aliases=['COMMON', 'LATIN']) - if confusable: - # if the character is confusable we replace it with the first prefered alias - c = confusable[0]['homoglyphs'][0]['c'] - unconfusable_name += c - return unconfusable_name diff --git a/liberapay/utils/unconfusable.py b/liberapay/utils/unconfusable.py new file mode 100644 index 0000000000..71e8f81985 --- /dev/null +++ b/liberapay/utils/unconfusable.py @@ -0,0 +1,11 @@ +from confusable_homoglyphs import confusables + +def unconfusable_string(name): + unconfusable_name = '' + for c in name: + confusable = confusables.is_confusable(c, preferred_aliases=['COMMON', 'LATIN']) + if confusable: + # if the character is confusable we replace it with the first prefered alias + c = confusable[0]['homoglyphs'][0]['c'] + unconfusable_name += c + return unconfusable_name diff --git a/tests/py/test_communities.py b/tests/py/test_communities.py index 3119d5004a..efe6b55004 100644 --- a/tests/py/test_communities.py +++ b/tests/py/test_communities.py @@ -112,14 +112,6 @@ def test_create_community_already_taken_is_case_insensitive(self): with self.assertRaises(CommunityAlreadyExists): Community.create('TeSt', self.alice.id) - def test_unconfusable(self): - self.assertEqual('user2', Community._unconfusable('user2')) - self.assertEqual('alice', Community._unconfusable('alice')) - latin_string = 'AlaskaJazz' - mixed_string = 'ΑlaskaJazz' - self.assertNotEqual(latin_string, mixed_string) - self.assertEqual(latin_string, Community._unconfusable(mixed_string)) - def test_create_community_already_taken_with_confusable_homoglyphs(self): latin_string = 'AlaskaJazz' mixed_string = 'ΑlaskaJazz' diff --git a/tests/py/test_utils.py b/tests/py/test_utils.py index cc12787f85..6a5cab5e15 100644 --- a/tests/py/test_utils.py +++ b/tests/py/test_utils.py @@ -7,7 +7,7 @@ from liberapay import utils from liberapay.i18n.currencies import Money, MoneyBasket from liberapay.testing import Harness -from liberapay.utils import markdown, b64encode_s, b64decode_s, cbor +from liberapay.utils import markdown, b64encode_s, b64decode_s, cbor, unconfusable from liberapay.wireup import CSP @@ -244,3 +244,14 @@ def test_csp_handles_valueless_directives_correctly(self): csp2 = CSP(csp) assert csp == csp2 assert csp2.directives[b'upgrade-insecure-requests'] == b'' + + # Unconfusable + # ============ + + def test_unconfusable_string(self): + self.assertEqual('user2', unconfusable.unconfusable_string('user2')) + self.assertEqual('alice', unconfusable.unconfusable_string('alice')) + latin_string = 'AlaskaJazz' + mixed_string = 'ΑlaskaJazz' + self.assertNotEqual(latin_string, mixed_string) + self.assertEqual(latin_string, unconfusable.unconfusable_string(mixed_string)) From 64033bddaf8e7e8ef071831dd9c976379f1b46f2 Mon Sep 17 00:00:00 2001 From: Martin Delille Date: Thu, 21 Mar 2019 15:34:36 +0100 Subject: [PATCH 6/9] disable confusable_homoglyphs update in make env --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index c43839c603..70692bf796 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,6 @@ env: requirements*.txt $(python) -m $(pip) install $(install_where) "virtualenv>=15.0.0" $(python) -m virtualenv --no-download ./env/ $(env_bin)/$(pip) install --require-hashes $$(for f in requirements_*.txt; do echo "-r $$f"; done) - $(env_bin)/python -c "from confusable_homoglyphs import cli; cli.update()" @touch env rehash-requirements: From 03b1fa39f628449ed1ccb2baeea4841d7c6fb231 Mon Sep 17 00:00:00 2001 From: Martin Delille Date: Thu, 21 Mar 2019 15:35:07 +0100 Subject: [PATCH 7/9] add spacing line in requirements_base.txt --- requirements_base.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements_base.txt b/requirements_base.txt index e924826d9d..4175dbeebf 100644 --- a/requirements_base.txt +++ b/requirements_base.txt @@ -243,6 +243,7 @@ cryptography==2.4.2 \ asn1crypto==0.24.0 \ --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \ --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 + confusable_homoglyphs==3.2.0 \ --hash=sha256:3b4a0d9fa510669498820c91a0bfc0c327568cecec90648cf3819d4a6fc6a751 \ --hash=sha256:e3ce611028d882b74a5faa69e3cbb5bd4dcd9f69936da6e73d33eda42c917944 From 2a96f25164ea687f67ee6a9915a485e9d3950f7b Mon Sep 17 00:00:00 2001 From: Martin Delille Date: Thu, 21 Mar 2019 15:35:34 +0100 Subject: [PATCH 8/9] one liner import --- tests/py/test_communities.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/py/test_communities.py b/tests/py/test_communities.py index efe6b55004..ab3ce5d98c 100644 --- a/tests/py/test_communities.py +++ b/tests/py/test_communities.py @@ -1,9 +1,6 @@ import json -from liberapay.exceptions import ( - AuthRequired, - CommunityAlreadyExists, -) +from liberapay.exceptions import AuthRequired, CommunityAlreadyExists from liberapay.models.community import Community from liberapay.testing import Harness From a73af9216ee836e46fb78c601c4baedfac4e341a Mon Sep 17 00:00:00 2001 From: Martin Delille Date: Fri, 22 Mar 2019 19:06:03 +0100 Subject: [PATCH 9/9] document unconfusable_string --- liberapay/utils/unconfusable.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/liberapay/utils/unconfusable.py b/liberapay/utils/unconfusable.py index 71e8f81985..98495186ae 100644 --- a/liberapay/utils/unconfusable.py +++ b/liberapay/utils/unconfusable.py @@ -1,11 +1,13 @@ from confusable_homoglyphs import confusables -def unconfusable_string(name): - unconfusable_name = '' - for c in name: +# Convert an Unicode string to its equivalent replacing all confusable homoglyphs +# to its common/latin equivalent +def unconfusable_string(s): + unconfusable_string = '' + for c in s: confusable = confusables.is_confusable(c, preferred_aliases=['COMMON', 'LATIN']) if confusable: # if the character is confusable we replace it with the first prefered alias c = confusable[0]['homoglyphs'][0]['c'] - unconfusable_name += c - return unconfusable_name + unconfusable_string += c + return unconfusable_string