From af4b34420e07e5c895011b721091113842d1d8be Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Thu, 22 Feb 2024 17:36:59 +0000 Subject: [PATCH 001/211] add make watch-tests command --- .gitignore | 1 + Makefile | 4 ++++ requirements_for_test.txt | 2 ++ 3 files changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 82c65ad35..880f6fcd0 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ nosetests.xml coverage.xml *,cover .pytest_cache/ +.testmondata* # Translations *.mo diff --git a/Makefile b/Makefile index 17964a497..66d208ade 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,10 @@ test: ## Run tests pytest -n auto python setup.py sdist +.PHONY: watch-tests +watch-tests: ## Automatically rerun tests + ptw --runner "pytest --testmon -n auto" + clean: rm -rf cache venv diff --git a/requirements_for_test.txt b/requirements_for_test.txt index caafecea9..e6c74fd25 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -11,6 +11,8 @@ freezegun==1.2.2 flake8-bugbear==22.10.27 flake8-print==5.0.0 pytest-profiling==1.7.0 +pytest-testmon==2.1.0 +pytest-watch==4.2.0 redis>=4.3.4 # Earlier versions of redis miss features the tests need snakeviz==2.1.1 black==23.10.1 # Also update `.pre-commit-config.yaml` if this changes From 8065a172053a554240bae3ebf0b7e5c8a13adc7f Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 8 Mar 2024 12:04:40 +0000 Subject: [PATCH 002/211] logging: split formatting-related things out into separate module with the intention that it's easily importable from the gunicorn main process without too many unintended additional imports --- .../{logging.py => logging/__init__.py} | 44 +++---------------- notifications_utils/logging/formatting.py | 43 ++++++++++++++++++ 2 files changed, 49 insertions(+), 38 deletions(-) rename notifications_utils/{logging.py => logging/__init__.py} (85%) create mode 100644 notifications_utils/logging/formatting.py diff --git a/notifications_utils/logging.py b/notifications_utils/logging/__init__.py similarity index 85% rename from notifications_utils/logging.py rename to notifications_utils/logging/__init__.py index 12ee225e2..bdedbf454 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging/__init__.py @@ -9,12 +9,14 @@ from flask import current_app, g, request from flask.ctx import has_app_context, has_request_context -from pythonjsonlogger.jsonlogger import JsonFormatter as BaseJSONFormatter -LOG_FORMAT = ( - "%(asctime)s %(app_name)s %(name)s %(levelname)s " '%(request_id)s "%(message)s" [in %(pathname)s:%(lineno)d]' +from .formatting import ( + LOG_FORMAT, + TIME_FORMAT, + BaseJSONFormatter, # noqa + Formatter, + JSONFormatter, ) -TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" logger = logging.getLogger(__name__) @@ -246,37 +248,3 @@ def user_id(self): def filter(self, record): record.user_id = self.user_id return record - - -class _MicrosecondAddingFormatterMixin: - """ - Appends a `.` and then a 6-digit number of microseconds to whatever - the superclass' `.formatTime(...)` returns. - """ - - # This is necessary because supplying a `datefmt` causes the base - # `formatTime` implementation to completely bypass any code that - # would be able to add milliseconds (let alone microseconds" to the - # formatted time. - - def formatTime(self, record, *args, **kwargs): - formatted = super().formatTime(record, *args, **kwargs) - return f"{formatted}.{int((record.created - int(record.created)) * 1e6):06}" - - -class Formatter(_MicrosecondAddingFormatterMixin, logging.Formatter): - pass - - -class JSONFormatter(_MicrosecondAddingFormatterMixin, BaseJSONFormatter): - def process_log_record(self, log_record): - rename_map = { - "asctime": "time", - "request_id": "requestId", - "app_name": "application", - "service_id": "service_id", - } - for key, newkey in rename_map.items(): - log_record[newkey] = log_record.pop(key) - log_record["logType"] = "application" - return log_record diff --git a/notifications_utils/logging/formatting.py b/notifications_utils/logging/formatting.py new file mode 100644 index 000000000..306da1c5c --- /dev/null +++ b/notifications_utils/logging/formatting.py @@ -0,0 +1,43 @@ +# This file is intentionally minimal to make it importable from gunicorn_config.py +import logging + +from pythonjsonlogger.jsonlogger import JsonFormatter as BaseJSONFormatter + +LOG_FORMAT = ( + "%(asctime)s %(app_name)s %(name)s %(levelname)s " '%(request_id)s "%(message)s" [in %(pathname)s:%(lineno)d]' +) +TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" + + +class _MicrosecondAddingFormatterMixin: + """ + Appends a `.` and then a 6-digit number of microseconds to whatever + the superclass' `.formatTime(...)` returns. + """ + + # This is necessary because supplying a `datefmt` causes the base + # `formatTime` implementation to completely bypass any code that + # would be able to add milliseconds (let alone microseconds" to the + # formatted time. + + def formatTime(self, record, *args, **kwargs): + formatted = super().formatTime(record, *args, **kwargs) + return f"{formatted}.{int((record.created - int(record.created)) * 1e6):06}" + + +class Formatter(_MicrosecondAddingFormatterMixin, logging.Formatter): + pass + + +class JSONFormatter(_MicrosecondAddingFormatterMixin, BaseJSONFormatter): + def process_log_record(self, log_record): + rename_map = { + "asctime": "time", + "request_id": "requestId", + "app_name": "application", + "service_id": "service_id", + } + for key, newkey in rename_map.items(): + log_record[newkey] = log_record.pop(key) + log_record["logType"] = "application" + return log_record From 67ac07602f11cba245c0d89a972aba26f04a3a66 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Mon, 25 Mar 2024 12:23:30 +0000 Subject: [PATCH 003/211] JSONFormatter: don't fail if non-standard log record attrs aren't found otherwise this won't work in cases our custom filters aren't being used (i.e. gunicorn main process) --- notifications_utils/logging/formatting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notifications_utils/logging/formatting.py b/notifications_utils/logging/formatting.py index 306da1c5c..92d6ecf29 100644 --- a/notifications_utils/logging/formatting.py +++ b/notifications_utils/logging/formatting.py @@ -38,6 +38,6 @@ def process_log_record(self, log_record): "service_id": "service_id", } for key, newkey in rename_map.items(): - log_record[newkey] = log_record.pop(key) + log_record[newkey] = log_record.pop(key, None) log_record["logType"] = "application" return log_record From e85d222b0fa76044abef8ede455e56d5da68632a Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Mon, 25 Mar 2024 12:19:18 +0000 Subject: [PATCH 004/211] logging init_app: tidy, comment root logger setup for clarity --- notifications_utils/logging/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/notifications_utils/logging/__init__.py b/notifications_utils/logging/__init__.py index bdedbf454..0ed7a5c32 100644 --- a/notifications_utils/logging/__init__.py +++ b/notifications_utils/logging/__init__.py @@ -97,10 +97,11 @@ def after_request(response): return response + app.logger.handlers.clear() + logging.getLogger().handlers.clear() + # avoid lastResort handler coming into play logging.getLogger().addHandler(logging.NullHandler()) - del app.logger.handlers[:] - if app.config["NOTIFY_RUNTIME_PLATFORM"] != "ecs": # TODO: ecs-migration: check if we still need this function after we migrate to ecs ensure_log_path_exists(app.config["NOTIFY_LOG_PATH"]) From ecb15886959ad139075ce671236396777e57a5a6 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Wed, 20 Mar 2024 15:02:10 +0000 Subject: [PATCH 005/211] add gunicorn_defaults module this contains a lot of common configuration we apply (or *should* apply) across all of our gunicorn servers. implemented as a globals-mutating function to allow it to make mutating changes to things beyond its module. indeed it is weird and a little bit ugly, but I think it's inevitable. includes new dict-based config for gunicorn's logging which will result in structured logs. --- notifications_utils/gunicorn_defaults.py | 76 ++++++++++++++++++++++++ setup.py | 1 + 2 files changed, 77 insertions(+) create mode 100644 notifications_utils/gunicorn_defaults.py diff --git a/notifications_utils/gunicorn_defaults.py b/notifications_utils/gunicorn_defaults.py new file mode 100644 index 000000000..b00922773 --- /dev/null +++ b/notifications_utils/gunicorn_defaults.py @@ -0,0 +1,76 @@ +# this will be imported at a very early stage of gunicorn initialization, so must +# be very restrained with imports + +import os +import sys +import traceback + +import gunicorn +from gunicorn.glogging import CONFIG_DEFAULTS as LOGGING_CONFIG_DEFAULTS + + +def on_starting(server): + notify_app_name = os.getenv("NOTIFY_APP_NAME") + server.log.info("Starting webapp %s", notify_app_name, extra={"notify_app_name": notify_app_name}) + + +def on_exit(server): + notify_app_name = os.getenv("NOTIFY_APP_NAME") + server.log.info("Stopping webapp %s", notify_app_name, extra={"notify_app_name": notify_app_name}) + + +def post_fork(server, worker): + import logging + + # near-silence messages generated before app has set its own logging up + for handler in logging.getLogger().handlers: + handler.setLevel(logging.ERROR) + + +def worker_int(worker): + worker.log.info("worker pid %s received SIGINT", worker.pid, extra={"process_": worker.pid}) + + +def worker_abort(worker): + worker.log.info("worker pid %s received ABORT", worker.pid, extra={"process_": worker.pid}) + for _threadId, stack in sys._current_frames().items(): + worker.log.error("".join(traceback.format_stack(stack))) + + +# a globals-mutating function because we need to update values in other modules, which +# isn't very nice to do at import-time +def set_gunicorn_defaults(globals_dict: dict): + globals_dict.update( + bind=f"0.0.0.0:{os.getenv('PORT', '8080')}", + disable_redirect_access_to_syslog=True, + logconfig_dict={ + **LOGGING_CONFIG_DEFAULTS, + "loggers": { + **LOGGING_CONFIG_DEFAULTS.get("loggers", {}), + "gunicorn.error": { + **LOGGING_CONFIG_DEFAULTS.get("loggers", {}).get("gunicorn.error", {}), + "propagate": False, # avoid duplicates + }, + "gunicorn.access": { + **LOGGING_CONFIG_DEFAULTS.get("loggers", {}).get("gunicorn.access", {}), + "level": "CRITICAL", + "propagate": False, + }, + }, + "formatters": { + **LOGGING_CONFIG_DEFAULTS.get("formatters", {}), + "generic": { + "class": "notifications_utils.logging.formatting.JSONFormatter", + "format": "ext://notifications_utils.logging.formatting.LOG_FORMAT", + "datefmt": "ext://notifications_utils.logging.formatting.TIME_FORMAT", + }, + }, + }, + on_exit=on_exit, + on_starting=on_starting, + post_fork=post_fork, + statsd_host="{}:8125".format(os.getenv("STATSD_HOST", "127.0.0.1")), + worker_abort=worker_abort, + worker_int=worker_int, + ) + gunicorn.SERVER_SOFTWARE = "None" diff --git a/setup.py b/setup.py index 07e3ceeaf..0e9a7b8da 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ "requests>=2.25.0", "python-json-logger>=2.0.1", "Flask>=2.1.1", + "gunicorn>=20.0.0", "ordered-set>=4.1.0", "Jinja2>=2.11.3", "statsd>=3.3.0", From 48106443f8954e86d0312c555ae5771cb7b0d9b9 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 28 Mar 2024 16:46:56 +0000 Subject: [PATCH 006/211] Bump version to 75.1.0 --- CHANGELOG.md | 6 ++++++ notifications_utils/version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 363660bea..82d1d40a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # CHANGELOG +## 75.1.0 + +* Split `logging.formatting` submodule out of `logging` module. All components should remain + accessible via the `logging` module, so this shouldn't affect existing code. +* Introduce `gunicorn_defaults` module. + ## 75.0.0 * BREAKING CHANGE: notifications_utils/clients/encryption/encryption_client has been removed. It diff --git a/notifications_utils/version.py b/notifications_utils/version.py index c60203d70..4723377a6 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "75.0.0" # 9f541edcecccf37d8e9604584f6d57a7 +__version__ = "75.1.0" # 573d3ee08c9f5b38fae9b8744b67edfc From e61c0ae1923dfb6db2fa1e4574ae1b2ca6a9c0bc Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 5 Apr 2024 16:34:50 +0100 Subject: [PATCH 007/211] gunicorn_defaults: don't set statsd_host in defaults not all apps have statsd --- notifications_utils/gunicorn_defaults.py | 1 - 1 file changed, 1 deletion(-) diff --git a/notifications_utils/gunicorn_defaults.py b/notifications_utils/gunicorn_defaults.py index b00922773..ad648d093 100644 --- a/notifications_utils/gunicorn_defaults.py +++ b/notifications_utils/gunicorn_defaults.py @@ -69,7 +69,6 @@ def set_gunicorn_defaults(globals_dict: dict): on_exit=on_exit, on_starting=on_starting, post_fork=post_fork, - statsd_host="{}:8125".format(os.getenv("STATSD_HOST", "127.0.0.1")), worker_abort=worker_abort, worker_int=worker_int, ) From 610e516a3088e8490d5cba2c1c9fa2eaa1ad798a Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 5 Apr 2024 16:36:29 +0100 Subject: [PATCH 008/211] Bump version to 75.1.1 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82d1d40a4..9469a8346 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 75.1.1 + +* Don't set `statsd_host` in `set_gunicorn_defaults` - not all apps have statsd. + ## 75.1.0 * Split `logging.formatting` submodule out of `logging` module. All components should remain diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 4723377a6..53d10fe5d 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "75.1.0" # 573d3ee08c9f5b38fae9b8744b67edfc +__version__ = "75.1.1" # 305e3c3dbf326076dba72b7ce5d33968 From 6e3d421a9efe7d8d3e4922d91a855eb8221e3471 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 11 Apr 2024 15:50:32 +0100 Subject: [PATCH 009/211] Add an `InsensitiveSet` This behaves like a regular `set` except it determines uniqueness based on the normalised value of the members. --- notifications_utils/insensitive_dict.py | 5 +++++ tests/test_insensitive_dict.py | 21 ++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/notifications_utils/insensitive_dict.py b/notifications_utils/insensitive_dict.py index 1d666ade2..ef0ed63b0 100644 --- a/notifications_utils/insensitive_dict.py +++ b/notifications_utils/insensitive_dict.py @@ -58,3 +58,8 @@ def make_key(original_key): if original_key is None: return None return original_key.translate(InsensitiveDict.KEY_TRANSLATION_TABLE).lower() + + +class InsensitiveSet(OrderedSet): + def __init__(self, iterable): + return super().__init__(InsensitiveDict.from_keys(iterable).values()) diff --git a/tests/test_insensitive_dict.py b/tests/test_insensitive_dict.py index bde490aef..0d2af10c5 100644 --- a/tests/test_insensitive_dict.py +++ b/tests/test_insensitive_dict.py @@ -1,8 +1,9 @@ from functools import partial import pytest +from ordered_set import OrderedSet -from notifications_utils.insensitive_dict import InsensitiveDict +from notifications_utils.insensitive_dict import InsensitiveDict, InsensitiveSet from notifications_utils.recipients import Cell, Row @@ -91,3 +92,21 @@ def test_maintains_insertion_order(): assert d.keys() == ["b", "a", "c"] d["BB"] = None assert d.keys() == ["b", "a", "c", "bb"] + + +def test_insensitive_set(): + assert InsensitiveSet( + [ + "foo", + "F o o ", + "F_O_O", + "B_A_R", + "B a r", + "bar", + ] + ) == OrderedSet( + [ + "F_O_O", + "bar", + ] + ) From f2ffb37c0ff98afcedc0d9c502a3659a87ff6e6a Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 11 Apr 2024 15:58:51 +0100 Subject: [PATCH 010/211] Preserve the first, not last item found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Normal `dict`s can’t have duplicate keys. `InsensitiveDict`s can receive duplicate items, where normalising two different keys would cause them to be the same. At the moment the default behaviour is to keep the last-seen value. This commit adds a flag to preserve the first-seen value, and not overwrite it. This is most relevant for how we’re going to use `InsensitiveSet`, where it makes more sense for the format of a placeholder shown to the user to be how it first occurs in a template. --- notifications_utils/insensitive_dict.py | 7 ++++--- tests/test_insensitive_dict.py | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/notifications_utils/insensitive_dict.py b/notifications_utils/insensitive_dict.py index ef0ed63b0..283314dfa 100644 --- a/notifications_utils/insensitive_dict.py +++ b/notifications_utils/insensitive_dict.py @@ -16,9 +16,10 @@ class InsensitiveDict(dict): KEY_TRANSLATION_TABLE = {ord(c): None for c in " _-"} - def __init__(self, row_dict): + def __init__(self, row_dict, overwrite_duplicates=True): for key, value in row_dict.items(): - self[key] = value + if overwrite_duplicates or key not in self: + self[key] = value @classmethod def from_keys(cls, keys): @@ -29,7 +30,7 @@ def from_keys(cls, keys): - it stores the original, unnormalised key as the value of the item so it can be retrieved later """ - return cls({key: key for key in keys}) + return cls({key: key for key in keys}, overwrite_duplicates=False) def keys(self): return OrderedSet(super().keys()) diff --git a/tests/test_insensitive_dict.py b/tests/test_insensitive_dict.py index 0d2af10c5..d8dab37ce 100644 --- a/tests/test_insensitive_dict.py +++ b/tests/test_insensitive_dict.py @@ -106,7 +106,19 @@ def test_insensitive_set(): ] ) == OrderedSet( [ - "F_O_O", - "bar", + "foo", + "B_A_R", ] ) + + +@pytest.mark.parametrize( + "extra_args, expected_dict", + ( + ({}, {"foo": 3}), + ({"overwrite_duplicates": True}, {"foo": 3}), + ({"overwrite_duplicates": False}, {"foo": 1}), + ), +) +def test_overwrite_duplicates(extra_args, expected_dict): + assert InsensitiveDict({"foo": 1, "FOO": 2, "f_o_o": 3}, **extra_args) == expected_dict From 5eb6cb4b61e640114c4169ae941a15b15ae3cf2b Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 11 Apr 2024 22:13:20 +0100 Subject: [PATCH 011/211] Minor version bump to 75.2.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9469a8346..e4e3271c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 75.2.0 + +* Add `InsensitiveSet` class (behaves like a normal set, but with uniqueness determined by normalised values) + ## 75.1.1 * Don't set `statsd_host` in `set_gunicorn_defaults` - not all apps have statsd. diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 53d10fe5d..572b027b8 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "75.1.1" # 305e3c3dbf326076dba72b7ce5d33968 +__version__ = "75.2.0" # dcb211bcd897f265625247221b449913 From 5b9be133d7c3100a22a9daa3d2cf87816b19fd6d Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Wed, 3 Apr 2024 14:40:07 +0100 Subject: [PATCH 012/211] logging: remove use of NOTIFY_RUNTIME_PLATFORM config parameter it is silly and now also redundant --- notifications_utils/logging/__init__.py | 17 +----- tests/test_logging.py | 69 +------------------------ 2 files changed, 2 insertions(+), 84 deletions(-) diff --git a/notifications_utils/logging/__init__.py b/notifications_utils/logging/__init__.py index 0ed7a5c32..bbfe6c8b4 100644 --- a/notifications_utils/logging/__init__.py +++ b/notifications_utils/logging/__init__.py @@ -44,12 +44,8 @@ def init_app(app, statsd_client=None, extra_filters: Sequence[logging.Filter] = app.config.setdefault("NOTIFY_LOG_LEVEL", "INFO") app.config.setdefault("NOTIFY_APP_NAME", "none") app.config.setdefault("NOTIFY_LOG_PATH", "./log/application.log") - app.config.setdefault("NOTIFY_RUNTIME_PLATFORM", None) app.config.setdefault("NOTIFY_LOG_DEBUG_PATH_LIST", {"/_status", "/metrics"}) - app.config.setdefault( - "NOTIFY_REQUEST_LOG_LEVEL", - "CRITICAL" if app.config["NOTIFY_RUNTIME_PLATFORM"] == "paas" else "NOTSET", - ) + app.config.setdefault("NOTIFY_REQUEST_LOG_LEVEL", "CRITICAL") @app.before_request def before_request(): @@ -102,10 +98,6 @@ def after_request(response): # avoid lastResort handler coming into play logging.getLogger().addHandler(logging.NullHandler()) - if app.config["NOTIFY_RUNTIME_PLATFORM"] != "ecs": - # TODO: ecs-migration: check if we still need this function after we migrate to ecs - ensure_log_path_exists(app.config["NOTIFY_LOG_PATH"]) - handlers = get_handlers(app, extra_filters=extra_filters) loglevel = logging.getLevelName(app.config["NOTIFY_LOG_LEVEL"]) loggers = [ @@ -157,13 +149,6 @@ def is_200_static_log(log): # stream json to stdout in all cases handlers.append(configure_handler(stream_handler, app, json_formatter, extra_filters=extra_filters)) - # TODO: ecs-migration: delete this when we migrate to ecs - # only write json to file if we're not running on ECS - if app.config["NOTIFY_RUNTIME_PLATFORM"] != "ecs": - # machine readable json to both file and stdout - file_handler = logging.handlers.WatchedFileHandler(filename=f"{app.config['NOTIFY_LOG_PATH']}.json") - handlers.append(configure_handler(file_handler, app, json_formatter, extra_filters=extra_filters)) - return handlers diff --git a/tests/test_logging.py b/tests/test_logging.py index 201b7d1d7..38eb28a7a 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,6 +1,5 @@ import json import logging as builtin_logging -import logging.handlers as builtin_logging_handlers import time from unittest import mock @@ -26,56 +25,13 @@ class App: assert not (tmpdir / "foo").exists() -@pytest.mark.parametrize( - "platform", - [ - "local", - "paas", - "something-else", - ], -) -def test_get_handlers_sets_up_logging_appropriately_without_debug_when_not_on_ecs(tmpdir, platform): - class TestFilter(builtin_logging.Filter): - def filter(self, record): - record.arbitrary_info = "some-extra-info" - return record - +def test_get_handlers_sets_up_logging_appropriately_without_debug(tmpdir): class App: config = { # make a tempfile called foo "NOTIFY_LOG_PATH": str(tmpdir / "foo"), "NOTIFY_APP_NAME": "bar", "NOTIFY_LOG_LEVEL": "ERROR", - "NOTIFY_RUNTIME_PLATFORM": platform, - } - debug = False - - app = App() - - handlers = logging.get_handlers(app, extra_filters=[TestFilter()]) - - assert len(handlers) == 2 - assert type(handlers[0]) == builtin_logging.StreamHandler - assert type(handlers[0].formatter) == logging.JSONFormatter - assert len(handlers[0].filters) == 6 - - assert type(handlers[1]) == builtin_logging_handlers.WatchedFileHandler - assert type(handlers[1].formatter) == logging.JSONFormatter - assert len(handlers[1].filters) == 6 - - dir_contents = tmpdir.listdir() - assert len(dir_contents) == 1 - assert dir_contents[0].basename == "foo.json" - - -def test_get_handlers_sets_up_logging_appropriately_without_debug_on_ecs(tmpdir): - class App: - config = { - # make a tempfile called foo - "NOTIFY_LOG_PATH": str(tmpdir / "foo"), - "NOTIFY_APP_NAME": "bar", - "NOTIFY_LOG_LEVEL": "ERROR", - "NOTIFY_RUNTIME_PLATFORM": "ecs", } debug = False @@ -107,7 +63,6 @@ class App: "NOTIFY_LOG_PATH": str(tmpdir / "foo"), "NOTIFY_APP_NAME": "bar", "NOTIFY_LOG_LEVEL": "INFO", - "NOTIFY_RUNTIME_PLATFORM": "ecs", } debug = False @@ -545,25 +500,3 @@ def test_app_request_logger_level_set(app_with_mocked_logger, level_name, expect logging.init_app(app) assert mock_req_logger.setLevel.call_args_list[-1] == mock.call(expected_level) - - -@pytest.mark.parametrize( - "rtplatform,expected_level", - ( - ("ecs", builtin_logging.NOTSET), - ("local", builtin_logging.NOTSET), - ("paas", builtin_logging.CRITICAL), - ), -) -def test_app_request_logger_level_defaults(app_with_mocked_logger, rtplatform, expected_level): - app = app_with_mocked_logger - mock_req_logger = mock.Mock( - spec=builtin_logging.Logger("flask.app.request"), - handlers=[], - ) - app.logger.getChild.side_effect = lambda name: mock_req_logger if name == "request" else mock.DEFAULT - - app.config["NOTIFY_RUNTIME_PLATFORM"] = rtplatform - logging.init_app(app) - - assert mock_req_logger.setLevel.call_args_list[-1] == mock.call(expected_level) From df22fe3200648df61d8e9d3ea4946dc76bb549c7 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 4 Apr 2024 09:55:27 +0100 Subject: [PATCH 013/211] logging: remove use of NOTIFY_LOG_PATH config parameter we no longer use it for anything --- notifications_utils/logging/__init__.py | 1 - tests/conftest.py | 6 +----- tests/test_logging.py | 13 +++---------- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/notifications_utils/logging/__init__.py b/notifications_utils/logging/__init__.py index bbfe6c8b4..ec9a08c5f 100644 --- a/notifications_utils/logging/__init__.py +++ b/notifications_utils/logging/__init__.py @@ -43,7 +43,6 @@ def _common_request_extra_log_context(): def init_app(app, statsd_client=None, extra_filters: Sequence[logging.Filter] = tuple()): app.config.setdefault("NOTIFY_LOG_LEVEL", "INFO") app.config.setdefault("NOTIFY_APP_NAME", "none") - app.config.setdefault("NOTIFY_LOG_PATH", "./log/application.log") app.config.setdefault("NOTIFY_LOG_DEBUG_PATH_LIST", {"/_status", "/metrics"}) app.config.setdefault("NOTIFY_REQUEST_LOG_LEVEL", "CRITICAL") diff --git a/tests/conftest.py b/tests/conftest.py index 7371cb2c7..fee8ae11e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,11 +49,7 @@ def app_with_mocked_logger(mocker, tmpdir): "flask.sansio.app.create_logger", return_value=mocker.Mock(spec=logging.Logger("flask.app"), handlers=[]), ) - yield from _create_app( - extra_config={ - "NOTIFY_LOG_PATH": str(tmpdir / "foo"), - } - ) + yield from _create_app() @pytest.fixture(scope="session") diff --git a/tests/test_logging.py b/tests/test_logging.py index 38eb28a7a..8df31b851 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -10,9 +10,9 @@ from notifications_utils.testing.comparisons import AnyStringMatching, RestrictedAny -def test_get_handlers_sets_up_logging_appropriately_with_debug(tmpdir): +def test_get_handlers_sets_up_logging_appropriately_with_debug(): class App: - config = {"NOTIFY_LOG_PATH": str(tmpdir / "foo"), "NOTIFY_APP_NAME": "bar", "NOTIFY_LOG_LEVEL": "ERROR"} + config = {"NOTIFY_APP_NAME": "bar", "NOTIFY_LOG_LEVEL": "ERROR"} debug = True app = App() @@ -22,14 +22,11 @@ class App: assert len(handlers) == 1 assert type(handlers[0]) == builtin_logging.StreamHandler assert type(handlers[0].formatter) == logging.Formatter - assert not (tmpdir / "foo").exists() -def test_get_handlers_sets_up_logging_appropriately_without_debug(tmpdir): +def test_get_handlers_sets_up_logging_appropriately_without_debug(): class App: config = { - # make a tempfile called foo - "NOTIFY_LOG_PATH": str(tmpdir / "foo"), "NOTIFY_APP_NAME": "bar", "NOTIFY_LOG_LEVEL": "ERROR", } @@ -43,8 +40,6 @@ class App: assert type(handlers[0]) == builtin_logging.StreamHandler assert type(handlers[0].formatter) == logging.JSONFormatter - assert not (tmpdir / "foo.json").exists() - @pytest.mark.parametrize( "frozen_time,logged_time", @@ -59,8 +54,6 @@ def test_log_timeformat_fractional_seconds(frozen_time, logged_time, tmpdir): class App: config = { - # make a tempfile called foo - "NOTIFY_LOG_PATH": str(tmpdir / "foo"), "NOTIFY_APP_NAME": "bar", "NOTIFY_LOG_LEVEL": "INFO", } From 598604bf89bf41bacf9c2ddfbd008c11679fbf14 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 4 Apr 2024 12:25:02 +0100 Subject: [PATCH 014/211] Bump version to 76.0.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4e3271c6..3a6abcd58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 76.0.0 + +* Remove use of `NOTIFY_RUNTIME_PLATFORM` and `NOTIFY_LOG_PATH` flask config parameters, which no longer did anything. Technically this only affects users if the consumed the paramter themselves and relied on utils code setting default values for them. + ## 75.2.0 * Add `InsensitiveSet` class (behaves like a normal set, but with uniqueness determined by normalised values) diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 572b027b8..d3d661d53 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "75.2.0" # dcb211bcd897f265625247221b449913 +__version__ = "76.0.0" # a1ac3717196c8b4513bb05f388195ce7 From 542bed5304ac5e3554326ebdf79b8656fb711813 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 16 Apr 2024 10:17:23 +0100 Subject: [PATCH 015/211] =?UTF-8?q?Reject=20Gibraltar=E2=80=99s=20postcode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > Gibraltar Post is currently developing a postcode system based on > United Kingdom of Great Britain and Northern Ireland postcode system. > > For the time being, mailers are kindly requested to use GX11 1AA as > generic postcode for Gibraltar https://www.upu.int/UPU/media/upu/PostalEntitiesFiles/addressingUnit/gibEn.pdf While Gibraltar is a British Overseas Territory it’s considered to be in ‘Europe Zone 3’ according to Royal Mail: https://www.royalmail.com/sending/international/country-guides#accordion-item-11113 We’ve also confirmed with DVLA that they would: - treat mail addressesed to Gibraltar as international - reject mail addressed UK-style address (postcode only) to Gibraltar Therefore we have 2 options: 1. normalise `GX11 1AA` to `Gibraltar` 2. reject addresses that end in `GX11 1AA` (either way an address that uses both `GX11 1AA` and `Gibraltar` will be fine, same as any country’s postcode system) This commit goes with option 2, because: - if Gibraltar introduces a system with more than 1 postcode this will get messy - option 1 would impact performance of validating large CSV files because of https://github.com/alphagov/notifications-utils/blob/01d60fcf670caf4ce23b651106e6374a9df82a38/notifications_utils/countries/__init__.py#L34-L38 --- notifications_utils/postal_address.py | 5 ++++- tests/test_postal_address.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/notifications_utils/postal_address.py b/notifications_utils/postal_address.py index 2bd387db3..231d7c33e 100644 --- a/notifications_utils/postal_address.py +++ b/notifications_utils/postal_address.py @@ -228,9 +228,12 @@ def normalise_postcode(postcode): def _is_a_real_uk_postcode(postcode): + normalised = normalise_postcode(postcode) + if normalised == "GX111AA": + return False # GIR0AA is Girobank pattern = re.compile(r"([A-Z]{1,2}[0-9][0-9A-Z]?[0-9][A-BD-HJLNP-UW-Z]{2})|(GIR0AA)") - return bool(pattern.fullmatch(normalise_postcode(postcode))) + return bool(pattern.fullmatch(normalised)) def format_postcode_for_printing(postcode): diff --git a/tests/test_postal_address.py b/tests/test_postal_address.py index 6f8288800..6949a6783 100644 --- a/tests/test_postal_address.py +++ b/tests/test_postal_address.py @@ -651,6 +651,9 @@ def test_normalise_postcode(postcode, normalised_postcode): # Giro Bank valid postcode and invalid postcode ("GIR0AA", True), ("GIR0AB", False), + # Gibraltar’s one postcode is not valid because it’s in the + # Europe postal zone + ("GX111AA", False), ], ) def test_if_postcode_is_a_real_uk_postcode(postcode, result): From 3e47f531b1d210a8dd1d4855f0c8e0b432fb818c Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 16 Apr 2024 10:29:31 +0100 Subject: [PATCH 016/211] Patch version bump to 76.0.1 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a6abcd58..31436a3f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 76.0.1 + +* Reject Gibraltar’s postcode (`GX11 1AA) when validating postal addresses + ## 76.0.0 * Remove use of `NOTIFY_RUNTIME_PLATFORM` and `NOTIFY_LOG_PATH` flask config parameters, which no longer did anything. Technically this only affects users if the consumed the paramter themselves and relied on utils code setting default values for them. diff --git a/notifications_utils/version.py b/notifications_utils/version.py index d3d661d53..ca46d876f 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "76.0.0" # a1ac3717196c8b4513bb05f388195ce7 +__version__ = "76.0.1" # 20bd59ac4566d156c93e64befbcc3fd8 From 22c16673e313b5210b36966005ee4b20e6cec4e6 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 16 Apr 2024 11:04:47 +0100 Subject: [PATCH 017/211] bump black and ruff --- .pre-commit-config.yaml | 4 ++-- requirements_for_test.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 66f06a833..be6f150c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,12 +7,12 @@ repos: - id: check-yaml - id: debug-statements - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.272' + rev: 'v0.3.7' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 24.4.0 hooks: - id: black name: black (python) diff --git a/requirements_for_test.txt b/requirements_for_test.txt index e6c74fd25..e989380c9 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -15,5 +15,5 @@ pytest-testmon==2.1.0 pytest-watch==4.2.0 redis>=4.3.4 # Earlier versions of redis miss features the tests need snakeviz==2.1.1 -black==23.10.1 # Also update `.pre-commit-config.yaml` if this changes -ruff==0.0.272 # Also update `.pre-commit-config.yaml` if this changes +black==24.4.0 # Also update `.pre-commit-config.yaml` if this changes +ruff==0.3.7 # Also update `.pre-commit-config.yaml` if this changes From 5f33d7c04a94150fea2d34b741ae2efcfa8db7ac Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 16 Apr 2024 11:06:43 +0100 Subject: [PATCH 018/211] update deprecated pyproject ruff flags --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6e2991ed3..f02fdfcea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ line-length = 120 target-version = "py39" -select = [ +lint.select = [ "E", # pycodestyle "W", # pycodestyle "F", # pyflakes @@ -15,7 +15,7 @@ select = [ "C90", # mccabe cyclomatic complexity "G", # flake8-logging-format ] -ignore = [] +lint.ignore = [] exclude = [ "migrations/versions/", "venv*", From ce619c18021e1a219cac2da4e3b0470ddfc08508 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 16 Apr 2024 11:07:13 +0100 Subject: [PATCH 019/211] apply ruff and black changes for latest versions --- notifications_utils/clients/antivirus/antivirus_client.py | 2 +- notifications_utils/field.py | 1 - notifications_utils/insensitive_dict.py | 1 - notifications_utils/serialised_model.py | 2 -- setup.py | 1 + 5 files changed, 2 insertions(+), 5 deletions(-) diff --git a/notifications_utils/clients/antivirus/antivirus_client.py b/notifications_utils/clients/antivirus/antivirus_client.py index c33f07294..f723db310 100644 --- a/notifications_utils/clients/antivirus/antivirus_client.py +++ b/notifications_utils/clients/antivirus/antivirus_client.py @@ -47,7 +47,7 @@ def scan(self, document_stream): error = AntivirusError.from_exception(e) current_app.logger.warning("Notify Antivirus API request failed with error: %s", error.message) - raise error + raise error from e finally: document_stream.seek(0) diff --git a/notifications_utils/field.py b/notifications_utils/field.py index 1567dd647..50df7e90a 100644 --- a/notifications_utils/field.py +++ b/notifications_utils/field.py @@ -48,7 +48,6 @@ def __repr__(self): class Field: - """ An instance of Field represents a string of text which may contain placeholders. diff --git a/notifications_utils/insensitive_dict.py b/notifications_utils/insensitive_dict.py index 283314dfa..7bfa01bc0 100644 --- a/notifications_utils/insensitive_dict.py +++ b/notifications_utils/insensitive_dict.py @@ -4,7 +4,6 @@ class InsensitiveDict(dict): - """ `InsensitiveDict` behaves like an ordered dictionary, except it normalises case, whitespace, hypens and underscores in keys. diff --git a/notifications_utils/serialised_model.py b/notifications_utils/serialised_model.py index dd2a3f66b..8e925008e 100644 --- a/notifications_utils/serialised_model.py +++ b/notifications_utils/serialised_model.py @@ -2,7 +2,6 @@ class SerialisedModel(ABC): - """ A SerialisedModel takes a dictionary, typically created by serialising a database object. It then takes the value of specified @@ -32,7 +31,6 @@ def __init__(self, _dict): class SerialisedModelCollection(ABC): - """ A SerialisedModelCollection takes a list of dictionaries, typically created by serialising database objects. When iterated over it diff --git a/setup.py b/setup.py index 0e9a7b8da..8c10b6ab5 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ """ Python API client for GOV.UK Notify """ + import ast import re From 47cefde1a75c1b99acf44808a3d3ab19f7ed663d Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 16 Apr 2024 11:09:43 +0100 Subject: [PATCH 020/211] ignore diffs --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index f7ca02cee..62d41727e 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,2 +1,3 @@ # bulk change for black formatting changes 0f3f4b3e6aa5a199a1b5e79742aa776ce8d8bf7e +ce619c18021e1a219cac2da4e3b0470ddfc08508 From ea48e7981b429c6bc7bc9e6b7888ea52386390d1 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 16 Apr 2024 11:11:53 +0100 Subject: [PATCH 021/211] patch version --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31436a3f6..ef76f6a95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 76.0.2 + +* linting changes + ## 76.0.1 * Reject Gibraltar’s postcode (`GX11 1AA) when validating postal addresses diff --git a/notifications_utils/version.py b/notifications_utils/version.py index ca46d876f..41c22d4f4 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "76.0.1" # 20bd59ac4566d156c93e64befbcc3fd8 +__version__ = "76.0.2" # 11145636cc878dc6d4c648ef30488612 From 8c1d83ec7abf0caaf295beaab79f3347d3be4a48 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Tue, 30 Apr 2024 09:20:28 +0100 Subject: [PATCH 022/211] Stop importing directly from `typing.re` This gets rid of the deprecation warning of "The typing.re namespace is deprecated and will be removed. These types should be directly imported from typing instead." --- notifications_utils/testing/comparisons.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notifications_utils/testing/comparisons.py b/notifications_utils/testing/comparisons.py index 58d133e9c..eb02b3a63 100644 --- a/notifications_utils/testing/comparisons.py +++ b/notifications_utils/testing/comparisons.py @@ -1,7 +1,7 @@ import re from functools import lru_cache from types import MappingProxyType -from typing.re import Pattern +from typing import Pattern class RestrictedAny: From 399037399e20a1f2a8b7eb7997cb29bc68bf5971 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Tue, 30 Apr 2024 10:23:03 +0100 Subject: [PATCH 023/211] Remove `check_proxy_header_before_request` from request_helper This was used to check the `X-Custom-Forwarder` header when running on the PaaS but is no longer needed. --- notifications_utils/request_helper.py | 38 +----------------- tests/test_request_header_authentication.py | 43 --------------------- 2 files changed, 1 insertion(+), 80 deletions(-) delete mode 100644 tests/test_request_header_authentication.py diff --git a/notifications_utils/request_helper.py b/notifications_utils/request_helper.py index 991eae6c8..e22fe75bc 100644 --- a/notifications_utils/request_helper.py +++ b/notifications_utils/request_helper.py @@ -1,7 +1,7 @@ from itertools import chain from random import SystemRandom -from flask import abort, current_app, request +from flask import current_app, request from flask.wrappers import Request @@ -153,39 +153,3 @@ def init_app(app): app.config["NOTIFY_TRACE_ID_HEADER"], app.config["NOTIFY_SPAN_ID_HEADER"], ) - - -def check_proxy_header_before_request(): - keys = [ - current_app.config.get("ROUTE_SECRET_KEY_1"), - current_app.config.get("ROUTE_SECRET_KEY_2"), - ] - result, msg = _check_proxy_header_secret(request, keys) - - if not result: - if current_app.config.get("CHECK_PROXY_HEADER", False): - current_app.logger.warning(msg) - abort(403) - - # We need to return None to continue processing the request - # http://flask.pocoo.org/docs/0.12/api/#flask.Flask.before_request - return None - - -def _check_proxy_header_secret(request, secrets, header="X-Custom-Forwarder"): - if header not in request.headers: - return False, "Header missing" - - header_secret = request.headers.get(header) - if not header_secret: - return False, "Header exists but is empty" - - # if there isn't any non-empty secret configured we fail closed - if not any(secrets): - return False, "Secrets are not configured" - - for i, secret in enumerate(secrets): - if header_secret == secret: - return True, f"Key used: {i + 1}" # add 1 to make it human-compatible - - return False, "Header didn't match any keys" diff --git a/tests/test_request_header_authentication.py b/tests/test_request_header_authentication.py deleted file mode 100644 index 81103ca1d..000000000 --- a/tests/test_request_header_authentication.py +++ /dev/null @@ -1,43 +0,0 @@ -from unittest import mock # noqa - -import pytest -from werkzeug.test import EnvironBuilder - -from notifications_utils.request_helper import NotifyRequest, _check_proxy_header_secret - - -@pytest.mark.parametrize( - "header,secrets,expected", - [ - ({"X-Custom-Forwarder": "right_key"}, ["right_key", "old_key"], (True, "Key used: 1")), - ({"X-Custom-Forwarder": "right_key"}, ["right_key"], (True, "Key used: 1")), - ({"X-Custom-Forwarder": "right_key"}, ["right_key", ""], (True, "Key used: 1")), - ({"My-New-Header": "right_key"}, ["right_key", ""], (True, "Key used: 1")), - ({"X-Custom-Forwarder": "right_key"}, ["", "right_key"], (True, "Key used: 2")), - ({"X-Custom-Forwarder": "right_key"}, ["", "old_key", "right_key"], (True, "Key used: 3")), - ({"X-Custom-Forwarder": ""}, ["right_key", "old_key"], (False, "Header exists but is empty")), - ({"X-Custom-Forwarder": "right_key"}, ["", None], (False, "Secrets are not configured")), - ({"X-Custom-Forwarder": "wrong_key"}, ["right_key", "old_key"], (False, "Header didn't match any keys")), - ], -) -def test_request_header_authorization(header, secrets, expected): - builder = EnvironBuilder() - builder.headers.extend(header) - request = NotifyRequest(builder.get_environ()) - - res = _check_proxy_header_secret(request, secrets, list(header.keys())[0]) - assert res == expected - - -@pytest.mark.parametrize( - "secrets,expected", - [ - (["old_key", "right_key"], (False, "Header missing")), - ], -) -def test_request_header_authorization_missing_header(secrets, expected): - builder = EnvironBuilder() - request = NotifyRequest(builder.get_environ()) - - res = _check_proxy_header_secret(request, secrets) - assert res == expected From c7b5fb5b93d04245488e01d435ffd43d4cbee104 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Tue, 30 Apr 2024 10:31:18 +0100 Subject: [PATCH 024/211] Bump version from 76.0.2 to 76.1.0 --- CHANGELOG.md | 5 +++++ notifications_utils/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef76f6a95..d2bcb8371 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG +## 76.1.0 + +* Remove `check_proxy_header_before_request` from request_helper.py since this was only used when apps + were deployed on the PaaS. + ## 76.0.2 * linting changes diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 41c22d4f4..c0aa8c75a 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "76.0.2" # 11145636cc878dc6d4c648ef30488612 +__version__ = "76.1.0" # 7ffeff9d42cce4ecf84336d77dbb92b6 From ca3784043816769f7ed41cd6ea992a89d5e8f710 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Fri, 31 May 2024 14:51:16 +0100 Subject: [PATCH 025/211] Update Zendesk client for new form We are switching to use a new Zendesk form, so this changes the Zendesk client. We now need to pass in the ID of the new form, and instead of populating the "Ticket category" dropdown we want to populate a dropdown called "Task type". This will only ever have one thing selected, so doesn't need to be able to take a list of values. --- .../clients/zendesk/zendesk_client.py | 10 ++++---- tests/clients/zendesk/test_zendesk_client.py | 24 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/notifications_utils/clients/zendesk/zendesk_client.py b/notifications_utils/clients/zendesk/zendesk_client.py index 20d57b527..f7774cb73 100644 --- a/notifications_utils/clients/zendesk/zendesk_client.py +++ b/notifications_utils/clients/zendesk/zendesk_client.py @@ -171,7 +171,7 @@ class NotifySupportTicket: NOTIFY_GROUP_ID = 360000036529 # Organization: GDS NOTIFY_ORG_ID = 21891972 - NOTIFY_TICKET_FORM_ID = 1900000284794 + NOTIFY_TICKET_FORM_ID = 14226867890588 def __init__( self, @@ -183,7 +183,7 @@ def __init__( user_email=None, requester_sees_message_content=True, notify_ticket_type: Optional[NotifyTicketType] = None, - ticket_categories=None, + notify_task_type=None, org_id=None, org_type=None, service_id=None, @@ -198,7 +198,7 @@ def __init__( self.user_email = user_email self.requester_sees_message_content = requester_sees_message_content self.notify_ticket_type = notify_ticket_type - self.ticket_categories = ticket_categories or [] + self.notify_task_type = notify_task_type self.org_id = org_id self.org_type = org_type self.service_id = service_id @@ -236,14 +236,14 @@ def request_data(self): def _get_custom_fields(self): org_type_tag = f"notify_org_type_{self.org_type}" if self.org_type else None custom_fields = [ - {"id": "360022836500", "value": self.ticket_categories}, # Notify Ticket category field + {"id": "14229641690396", "value": self.notify_task_type}, # Notify Task type field {"id": "360022943959", "value": self.org_id}, # Notify Organisation ID field {"id": "360022943979", "value": org_type_tag}, # Notify Organisation type field {"id": "1900000745014", "value": self.service_id}, # Notify Service ID field ] if self.notify_ticket_type: - # Notify Ticket type field + # Notify Responder field custom_fields.append({"id": "1900000744994", "value": self.notify_ticket_type.value}) return custom_fields diff --git a/tests/clients/zendesk/test_zendesk_client.py b/tests/clients/zendesk/test_zendesk_client.py index c7d68f6bd..0c40f25cc 100644 --- a/tests/clients/zendesk/test_zendesk_client.py +++ b/tests/clients/zendesk/test_zendesk_client.py @@ -123,7 +123,7 @@ def test_notify_support_ticket_request_data(p1_arg, expected_tags, expected_prio "tags": expected_tags, "type": "question", "custom_fields": [ - {"id": "360022836500", "value": []}, + {"id": "14229641690396", "value": None}, {"id": "360022943959", "value": None}, {"id": "360022943979", "value": None}, {"id": "1900000745014", "value": None}, @@ -149,21 +149,21 @@ def test_notify_support_ticket_request_data_with_user_name_and_email(name, zende @pytest.mark.parametrize( - "custom_fields, tech_ticket_tag, categories, org_id, org_type, service_id", + "custom_fields, tech_ticket_tag, notify_task_type, org_id, org_type, service_id", [ - ({"notify_ticket_type": NotifyTicketType.TECHNICAL}, "notify_ticket_type_technical", [], None, None, None), + ({"notify_ticket_type": NotifyTicketType.TECHNICAL}, "notify_ticket_type_technical", None, None, None, None), ( {"notify_ticket_type": NotifyTicketType.NON_TECHNICAL}, "notify_ticket_type_non_technical", - [], + None, None, None, None, ), ( - {"ticket_categories": ["notify_billing", "notify_bug"]}, + {"notify_task_type": "notify_task_email_branding"}, None, - ["notify_billing", "notify_bug"], + "notify_task_email_branding", None, None, None, @@ -171,7 +171,7 @@ def test_notify_support_ticket_request_data_with_user_name_and_email(name, zende ( {"org_id": "1234", "org_type": "local"}, None, - [], + None, "1234", "notify_org_type_local", None, @@ -179,7 +179,7 @@ def test_notify_support_ticket_request_data_with_user_name_and_email(name, zende ( {"service_id": "abcd", "org_type": "nhs"}, None, - [], + None, None, "notify_org_type_nhs", "abcd", @@ -189,7 +189,7 @@ def test_notify_support_ticket_request_data_with_user_name_and_email(name, zende def test_notify_support_ticket_request_data_custom_fields( custom_fields, tech_ticket_tag, - categories, + notify_task_type, org_id, org_type, service_id, @@ -200,7 +200,9 @@ def test_notify_support_ticket_request_data_custom_fields( assert {"id": "1900000744994", "value": tech_ticket_tag} in notify_ticket_form.request_data["ticket"][ "custom_fields" ] - assert {"id": "360022836500", "value": categories} in notify_ticket_form.request_data["ticket"]["custom_fields"] + assert {"id": "14229641690396", "value": notify_task_type} in notify_ticket_form.request_data["ticket"][ + "custom_fields" + ] assert {"id": "360022943959", "value": org_id} in notify_ticket_form.request_data["ticket"]["custom_fields"] assert {"id": "360022943979", "value": org_type} in notify_ticket_form.request_data["ticket"]["custom_fields"] assert {"id": "1900000745014", "value": service_id} in notify_ticket_form.request_data["ticket"]["custom_fields"] @@ -231,7 +233,7 @@ def test_notify_support_ticket_with_html_body(): "tags": ["govuk_notify_support"], "type": "task", "custom_fields": [ - {"id": "360022836500", "value": []}, + {"id": "14229641690396", "value": None}, {"id": "360022943959", "value": None}, {"id": "360022943979", "value": None}, {"id": "1900000745014", "value": None}, From 61ba934e16fc98016593a70344fa26bf11889666 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Fri, 31 May 2024 14:56:28 +0100 Subject: [PATCH 026/211] Major version bump to `77.0.0` --- CHANGELOG.md | 5 +++++ notifications_utils/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2bcb8371..04d406dee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG +## 77.0.0 + +* Breaking change to the Zendesk Client. The `ticket_categories` argument has been replaced with a `notify_task_type` argument, and it now populates + a different Zendesk form. + ## 76.1.0 * Remove `check_proxy_header_before_request` from request_helper.py since this was only used when apps diff --git a/notifications_utils/version.py b/notifications_utils/version.py index c0aa8c75a..5e548c532 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "76.1.0" # 7ffeff9d42cce4ecf84336d77dbb92b6 +__version__ = "77.0.0" # 72d8310e6b5e4ead204ca33678e9f84b From c56868c9f6a41393c5161b55cc263ac2d53583f4 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 9 May 2024 09:49:03 +0100 Subject: [PATCH 027/211] Add a command to copy utils pyproject.toml into apps This will let us share linter config between the apps. In the future it might even let us share common dependencies between the apps, since pyproject.toml can do dependency management. This will require adding a `make` command to each app like: ```bash python -c "from notifications_utils.version_tools import copy_pyproject_toml; copy_pyproject_toml()" ```` --- notifications_utils/version_tools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/notifications_utils/version_tools.py b/notifications_utils/version_tools.py index 4dbd22f62..18432a697 100644 --- a/notifications_utils/version_tools.py +++ b/notifications_utils/version_tools.py @@ -3,6 +3,7 @@ import requests requirements_file = pathlib.Path("requirements.in") +pyproject_file = pathlib.Path("pyproject.toml") repo_name = "alphagov/notifications-utils" @@ -76,3 +77,9 @@ def get_relevant_changelog_lines(current_version, newest_version): def get_file_contents_from_github(branch_or_tag, path): return requests.get(f"https://raw.githubusercontent.com/{repo_name}/{branch_or_tag}/{path}").text + + +def copy_pyproject_toml(): + local_utils_version = get_app_version() + remote_contents = get_file_contents_from_github(local_utils_version, "pyproject.toml") + pyproject_file.write_text(remote_contents) From c20f005166d6b18f3478fd6507a67476f5d77cd8 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 30 May 2024 17:35:02 +0100 Subject: [PATCH 028/211] Increase minimum Python version (we are running 3.11 everywhere now) --- notifications_utils/recipients.py | 2 +- pyproject.toml | 2 +- tests/test_sanitise_text.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index 55ec4cff4..d8b0aa413 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -164,7 +164,7 @@ def get_rows(self): output_dict = {} - for column_name, column_value in zip(column_headers, row): + for column_name, column_value in zip(column_headers, row, strict=False): column_value = strip_and_remove_obscure_whitespace(column_value) if InsensitiveDict.make_key(column_name) in self.recipient_column_headers_as_column_keys: diff --git a/pyproject.toml b/pyproject.toml index f02fdfcea..ce1ab3c9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ line-length = 120 [tool.ruff] line-length = 120 -target-version = "py39" +target-version = "py311" lint.select = [ "E", # pycodestyle diff --git a/tests/test_sanitise_text.py b/tests/test_sanitise_text.py index db430cfe4..87eda9ec5 100644 --- a/tests/test_sanitise_text.py +++ b/tests/test_sanitise_text.py @@ -21,6 +21,7 @@ # this unicode char is not decomposable (("😬", "?"), "undecomposable unicode char (grimace emoji)"), (("↉", "?"), "vulgar fraction (↉) that we do not try decomposing"), + strict=True, ) @@ -43,6 +44,7 @@ def test_encode_chars_the_same_for_ascii_and_sms(char, expected, cls): (("ë", "ë", "e"), "non-gsm Welsh char (e with dots)"), (("Ò", "Ò", "O"), "non-gsm Welsh char (capital O with grave accent)"), (("í", "í", "i"), "non-gsm Welsh char (i with accent)"), + strict=True, ) From 47e527742191d6b0a0fe462a90cd3b5158342e6b Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 30 May 2024 17:47:17 +0100 Subject: [PATCH 029/211] Add a comment to the autogenerated file --- notifications_utils/version_tools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/notifications_utils/version_tools.py b/notifications_utils/version_tools.py index 18432a697..874d89b60 100644 --- a/notifications_utils/version_tools.py +++ b/notifications_utils/version_tools.py @@ -82,4 +82,6 @@ def get_file_contents_from_github(branch_or_tag, path): def copy_pyproject_toml(): local_utils_version = get_app_version() remote_contents = get_file_contents_from_github(local_utils_version, "pyproject.toml") - pyproject_file.write_text(remote_contents) + pyproject_file.write_text( + f"# This file is automatically copied from notifications-utils@{local_utils_version}\n\n{remote_contents}" + ) From 73fe00e69baf5b00f9a9adc7666b030f8c6138d3 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 8 May 2024 16:54:46 +0100 Subject: [PATCH 030/211] Re-enable flake8-print MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don’t want print statements in production code – should be using logging instead. Ruff doesn’t have this check on by default[1] so we need to explicitly include it. Some existing print statements in this codebase are there intentionally (they are in a command not a public endpoint) so I’ve marked them as exempt from checking by running `ruff check --add-noqa` Adding this now so our linting rules have parity with the admin app, which is the first app we’re going to copy the utils version of `pyproject.toml` into. 1. https://docs.astral.sh/ruff/rules/print/ --- notifications_utils/version_tools.py | 2 +- pyproject.toml | 1 + scripts/bump_version.py | 2 +- tests/clients/redis/test_redis_client.py | 5 ++--- tests/test_logging.py | 2 -- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/notifications_utils/version_tools.py b/notifications_utils/version_tools.py index 874d89b60..49b489a13 100644 --- a/notifications_utils/version_tools.py +++ b/notifications_utils/version_tools.py @@ -26,7 +26,7 @@ def upgrade_version(): write_version_to_requirements_file(newest_version) - print( + print( # noqa: T201 f"{color.GREEN}✅ {color.BOLD}notifications-utils bumped to {newest_version}{color.END}\n\n" f"{color.YELLOW}{color.UNDERLINE}Now run:{color.END}\n\n" f" make freeze-requirements\n" diff --git a/pyproject.toml b/pyproject.toml index ce1ab3c9a..afeb9532a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ lint.select = [ "B", # flake8-bugbear "C90", # mccabe cyclomatic complexity "G", # flake8-logging-format + "T20", # flake8-print ] lint.ignore = [] exclude = [ diff --git a/scripts/bump_version.py b/scripts/bump_version.py index 7c9b6ce40..b814446e7 100755 --- a/scripts/bump_version.py +++ b/scripts/bump_version.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# ruff: noqa: T201 import argparse import hashlib @@ -20,7 +21,6 @@ parser = argparse.ArgumentParser() parser.add_argument("version_part", choices=version_parts) version_part = parser.parse_args().version_part - current_major, current_minor, current_patch = map(int, old_version.split(".")) print(f"current version {old_version=}") diff --git a/tests/clients/redis/test_redis_client.py b/tests/clients/redis/test_redis_client.py index 10fdb7d26..78042abd7 100644 --- a/tests/clients/redis/test_redis_client.py +++ b/tests/clients/redis/test_redis_client.py @@ -237,7 +237,6 @@ def test_get_redis_lock_returns_stub_if_redis_not_enabled(mocked_redis_client): def test_redis_stub_lock_function_signatures_match(): - print(f"Testing StubLock agains redis=={redis.__version__}") lock_methods = dict(inspect.getmembers(redis.lock.Lock, inspect.isfunction)) stub_methods = dict(inspect.getmembers(StubLock, inspect.isfunction)) @@ -252,12 +251,12 @@ def test_redis_stub_lock_function_signatures_match(): } missing_methods = lock_methods_to_test - stub_methods.keys() - assert not missing_methods, "stub has missing methods" + assert not missing_methods, f"StubLock has missing methods (testing against redis=={redis.__version__})" for fn_name in lock_methods_to_test: lock_sig = inspect.signature(lock_methods[fn_name]) stub_sig = inspect.signature(stub_methods[fn_name]) - assert lock_sig.parameters == stub_sig.parameters + assert lock_sig.parameters == stub_sig.parameters, f"(testing StubLock against redis=={redis.__version__})" def test_decrby(mocked_redis_client): diff --git a/tests/test_logging.py b/tests/test_logging.py index 8df31b851..29caff0b5 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -287,8 +287,6 @@ def metrics(): app.test_client().get("/_status") - print(mock_req_logger.log.call_args_list) - assert ( mock.call( builtin_logging.DEBUG, From e9207df9129c9634170fca472ca50f13c9d4d34a Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 9 May 2024 09:51:35 +0100 Subject: [PATCH 031/211] Minor version bump to 76.2.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d406dee..b19e74d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 77.1.0 + +* Add `version_tools.copy_pyproject_toml` to share linter config between apps + ## 77.0.0 * Breaking change to the Zendesk Client. The `ticket_categories` argument has been replaced with a `notify_task_type` argument, and it now populates diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 5e548c532..743082a46 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "77.0.0" # 72d8310e6b5e4ead204ca33678e9f84b +__version__ = "77.1.0" # 0547464f2c9c5dde9cb473dfc2b166a7 From 91ee6c548a874f117ba30deb0ecd2091a2c1a253 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 3 Jun 2024 14:57:12 +0100 Subject: [PATCH 032/211] Look at `requirements.txt` to determine installed version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is better than looking at `requirements.in` because: - `requirements.in` isn’t always present (for example when running apps in Docker) - `requirements.txt` is more likely to reflect what’s actually installed (since that’s what we tell `pip` to look at) --- notifications_utils/version_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/notifications_utils/version_tools.py b/notifications_utils/version_tools.py index 49b489a13..29fb485dd 100644 --- a/notifications_utils/version_tools.py +++ b/notifications_utils/version_tools.py @@ -3,6 +3,7 @@ import requests requirements_file = pathlib.Path("requirements.in") +frozen_requirements_file = pathlib.Path("requirements.txt") pyproject_file = pathlib.Path("pyproject.toml") repo_name = "alphagov/notifications-utils" @@ -44,7 +45,7 @@ def get_remote_version(): def get_app_version(): - return next(line.split("@")[-1] for line in requirements_file.read_text().split("\n") if repo_name in line) + return next(line.split("@")[-1] for line in frozen_requirements_file.read_text().split("\n") if repo_name in line) def write_version_to_requirements_file(version): From de17049483842a718a18b83992057be4c75c4955 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 3 Jun 2024 15:00:12 +0100 Subject: [PATCH 033/211] Patch version bump to 77.1.1 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b19e74d2b..aa0d8b6c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 77.1.1 + +* Fix how `version_tools.copy_pyproject_toml` discovers the current version of notifications-utils + ## 77.1.0 * Add `version_tools.copy_pyproject_toml` to share linter config between apps diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 743082a46..63b558538 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "77.1.0" # 0547464f2c9c5dde9cb473dfc2b166a7 +__version__ = "77.1.1" # 07e8998aff89b28962f71a42867067ce From bc48366ef771e02bffed479fbcc2b1408330071c Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 19 Apr 2024 12:25:55 +0100 Subject: [PATCH 034/211] NotifyTask: include pid and other structured fields in completion logs --- notifications_utils/celery.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/notifications_utils/celery.py b/notifications_utils/celery.py index b6e52331c..cd7953d22 100644 --- a/notifications_utils/celery.py +++ b/notifications_utils/celery.py @@ -1,5 +1,6 @@ import time from contextlib import contextmanager +from os import getpid from celery import Celery, Task from flask import g, request @@ -39,7 +40,13 @@ def on_success(self, retval, task_id, args, kwargs): self.name, self.queue_name, elapsed_time, - extra={"time_taken": elapsed_time}, + extra={ + "celery_task": self.name, + "queue_name": self.queue_name, + "time_taken": elapsed_time, + # avoid name collision with LogRecord's own `process` attribute + "process_": getpid(), + }, ) app.statsd_client.timing( @@ -57,7 +64,13 @@ def on_failure(self, exc, task_id, args, kwargs, einfo): self.name, self.queue_name, elapsed_time, - extra={"time_taken": elapsed_time}, + extra={ + "celery_task": self.name, + "queue_name": self.queue_name, + "time_taken": elapsed_time, + # avoid name collision with LogRecord's own `process` attribute + "process_": getpid(), + }, ) app.statsd_client.incr(f"celery.{self.queue_name}.{self.name}.failure") From 50632d4eee2b4a2ee413c1d0337ec9366b9e4fa3 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 19 Apr 2024 12:29:24 +0100 Subject: [PATCH 035/211] Bump version to 77.2.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa0d8b6c4..149540c53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 77.2.0 + +* `NotifyTask`: include pid and other structured fields in completion log messages + ## 77.1.1 * Fix how `version_tools.copy_pyproject_toml` discovers the current version of notifications-utils diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 63b558538..0b9dcb1fd 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "77.1.1" # 07e8998aff89b28962f71a42867067ce +__version__ = "77.2.0" # dbe1d856722af4ad0303161298e2aba3 From b745ec22f35cb10578b68b7696fef7800603caea Mon Sep 17 00:00:00 2001 From: David McDonald Date: Tue, 21 May 2024 18:51:04 +0100 Subject: [PATCH 036/211] Ensure no stale data in Redis if Redis fails to delete cache key For context, when the admin app wants to get data from our API it will first try and get the data from redis and if the data doesn't exist in redis, it will get the data from the API and then finish by setting that data in redis. When the admin app wants to update data for our API, it currently calls out to the API to update the data and then deletes any existing/relevant data cached in redis. The subsequent time it tries to get this data that it just updated, it will use the usual approach for getting data that will set the redis key too. There is a problem with when the admin app wants to update something in the database. At the moment, it will start by calling the API to update the database and if that is successful it will then attempt to delete the relevant cache keys in redis. But redis may not always be available and in that case, it will - fail to delete the cache key - catch and ignore the exception raised when trying to delete the redis cache key This leads to the change the user requested being made in the database, but the cache still has the old data in it! This is bad because our apps check the cache first and this can result in us sending out incorrect emails using old templates, for example. We have a runbook to manually recover from this position if redis has downtime and delete queries fail during this time: https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-redis-outages Note, there is no issue with `cache.set` calls. If the call to redis fails as part of this, no stale data is produced. The database hasn't been changed and either redis hasn't been changed or hasn't had the current data added to it. This commit changes our caching logic so that we can't end up with stale data in redis if a delete to redis fails. If a delete to redis fails, then we error early and don't attempt to update the database. The trade off we make here is that now a user will see an error page if their request failed to delete from redis, whereas before they would have gotten a 200 (but ended up with stale data). We think it is worse to have stale data, then it is to fail a users request. --- .../clients/redis/request_cache.py | 26 ++++++++-------- tests/clients/redis/test_request_cache.py | 30 +++++++++---------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/notifications_utils/clients/redis/request_cache.py b/notifications_utils/clients/redis/request_cache.py index 47d282bd1..13d533740 100644 --- a/notifications_utils/clients/redis/request_cache.py +++ b/notifications_utils/clients/redis/request_cache.py @@ -68,12 +68,13 @@ def delete(self, key_format): def _delete(client_method): @wraps(client_method) def new_client_method(*args, **kwargs): - try: - api_response = client_method(*args, **kwargs) - finally: - redis_key = self._make_key(key_format, client_method, args, kwargs) - self.redis_client.delete(redis_key) - return api_response + # It is important to attempt the redis deletion first and raise an exception + # if it is unsuccessful. If we didn't, then we risk having a successful API + # call that updates the database, but redis left with stale data. Stale data + # is worse then failing the users requests + redis_key = self._make_key(key_format, client_method, args, kwargs) + self.redis_client.delete(redis_key, raise_exception=True) + return client_method(*args, **kwargs) return new_client_method @@ -83,12 +84,13 @@ def delete_by_pattern(self, key_format): def _delete(client_method): @wraps(client_method) def new_client_method(*args, **kwargs): - try: - api_response = client_method(*args, **kwargs) - finally: - redis_key = self._make_key(key_format, client_method, args, kwargs) - self.redis_client.delete_by_pattern(redis_key) - return api_response + # It is important to attempt the redis deletion first and raise an exception + # if it is unsuccessful. If we didn't, then we risk having a successful API + # call that updates the database, but redis left with stale data. Stale data + # is worse then failing the users requests + redis_key = self._make_key(key_format, client_method, args, kwargs) + self.redis_client.delete_by_pattern(redis_key, raise_exception=True) + return client_method(*args, **kwargs) return new_client_method diff --git a/tests/clients/redis/test_request_cache.py b/tests/clients/redis/test_request_cache.py index 258eff78b..22e3e8224 100644 --- a/tests/clients/redis/test_request_cache.py +++ b/tests/clients/redis/test_request_cache.py @@ -1,3 +1,5 @@ +from unittest.mock import MagicMock + import pytest from notifications_utils.clients.redis import RequestCache @@ -180,23 +182,21 @@ def foo(a, b, c): assert foo(*args) == "bar" - mock_redis_delete.assert_called_once_with(expected_cache_key) + mock_redis_delete.assert_called_once_with(expected_cache_key, raise_exception=True) -def test_delete_even_if_call_raises(mocker, mocked_redis_client, cache): - mock_redis_delete = mocker.patch.object( - mocked_redis_client, - "delete", - ) +def test_doesnt_update_api_if_redis_delete_fails(mocker, mocked_redis_client, cache): + mocker.patch.object(mocked_redis_client, "delete", side_effect=RuntimeError("API update failed")) + fake_api_call = MagicMock() @cache.delete("bar") def foo(): - raise RuntimeError + return fake_api_call() with pytest.raises(RuntimeError): foo() - mock_redis_delete.assert_called_once_with("bar") + fake_api_call.assert_not_called() def test_delete_by_pattern(mocker, mocked_redis_client, cache): @@ -211,20 +211,18 @@ def foo(a, b, c): assert foo(1, 2, 3) == "bar" - mock_redis_delete.assert_called_once_with("1-2-3-???") + mock_redis_delete.assert_called_once_with("1-2-3-???", raise_exception=True) -def test_delete_by_pattern_even_if_call_raises(mocker, mocked_redis_client, cache): - mock_redis_delete = mocker.patch.object( - mocked_redis_client, - "delete_by_pattern", - ) +def test_doesnt_update_api_if_redis_delete_by_pattern_fails(mocker, mocked_redis_client, cache): + mocker.patch.object(mocked_redis_client, "delete_by_pattern", side_effect=RuntimeError("API update failed")) + fake_api_call = MagicMock() @cache.delete_by_pattern("bar-???") def foo(): - raise RuntimeError + return fake_api_call() with pytest.raises(RuntimeError): foo() - mock_redis_delete.assert_called_once_with("bar-???") + fake_api_call.assert_not_called() From 1e14b310d4fa220a704bcfd6a0423dbf71ea8917 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 22 May 2024 13:59:57 +0100 Subject: [PATCH 037/211] Try second redis delete when updating API resource to avoid race In the previous commit, we tried to avoid stale data in redis if redis is unavailable. However, this introduced a race condition where another request re-populates the redis key with the old value before the database has been updated. We fix this by also deleting the cache key after the API update so that if that happened, we would remove the stale data after the API update. There is a small edge case that could still end up with stale data that this commit doesn't solve (and we may just have to tolerate). If we hit the race condition above so stale data is reinserted in redis before the database update, then the database update happens and redis goes down at this point. Then what will happen is the second cache clear will fail and we will be left with stale data. This should be a rare case as involves both a race condition happening so two requests for the same object at very similar times, and redis going down mid request. Again, we lean towards throwing an uncaught exception if a redis delete fails. It means there is a chance of stale data and the user should hopefully check if their action was successful and retry if they see stale data in the user interface (which should then hopefully fix the problem). --- .../clients/redis/request_cache.py | 24 +++++++++++++------ tests/clients/redis/test_request_cache.py | 8 ++++--- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/notifications_utils/clients/redis/request_cache.py b/notifications_utils/clients/redis/request_cache.py index 13d533740..d4fec2c53 100644 --- a/notifications_utils/clients/redis/request_cache.py +++ b/notifications_utils/clients/redis/request_cache.py @@ -68,13 +68,23 @@ def delete(self, key_format): def _delete(client_method): @wraps(client_method) def new_client_method(*args, **kwargs): + redis_key = self._make_key(key_format, client_method, args, kwargs) + # It is important to attempt the redis deletion first and raise an exception # if it is unsuccessful. If we didn't, then we risk having a successful API # call that updates the database, but redis left with stale data. Stale data # is worse then failing the users requests - redis_key = self._make_key(key_format, client_method, args, kwargs) self.redis_client.delete(redis_key, raise_exception=True) - return client_method(*args, **kwargs) + + api_response = client_method(*args, **kwargs) + + # We also attempt another redis deletion after the API. This is to deal with + # the race condition where another request repopulates redis with the old data + # before the database has been updated. We want to raise an exception if the call + # to redis here fails because that will hopefully prompt the user that something + # went wrong and they should retry their action (hopefully resolving the problem). + self.redis_client.delete(redis_key, raise_exception=True) + return api_response return new_client_method @@ -84,13 +94,13 @@ def delete_by_pattern(self, key_format): def _delete(client_method): @wraps(client_method) def new_client_method(*args, **kwargs): - # It is important to attempt the redis deletion first and raise an exception - # if it is unsuccessful. If we didn't, then we risk having a successful API - # call that updates the database, but redis left with stale data. Stale data - # is worse then failing the users requests + # See equivalent comments above for why we attempt the redis delete before and + # after the API call redis_key = self._make_key(key_format, client_method, args, kwargs) self.redis_client.delete_by_pattern(redis_key, raise_exception=True) - return client_method(*args, **kwargs) + api_response = client_method(*args, **kwargs) + self.redis_client.delete_by_pattern(redis_key, raise_exception=True) + return api_response return new_client_method diff --git a/tests/clients/redis/test_request_cache.py b/tests/clients/redis/test_request_cache.py index 22e3e8224..bdaa9587f 100644 --- a/tests/clients/redis/test_request_cache.py +++ b/tests/clients/redis/test_request_cache.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock +from unittest.mock import MagicMock, call import pytest @@ -182,7 +182,8 @@ def foo(a, b, c): assert foo(*args) == "bar" - mock_redis_delete.assert_called_once_with(expected_cache_key, raise_exception=True) + expected_call = call(expected_cache_key, raise_exception=True) + mock_redis_delete.assert_has_calls([expected_call, expected_call]) def test_doesnt_update_api_if_redis_delete_fails(mocker, mocked_redis_client, cache): @@ -211,7 +212,8 @@ def foo(a, b, c): assert foo(1, 2, 3) == "bar" - mock_redis_delete.assert_called_once_with("1-2-3-???", raise_exception=True) + expected_call = call("1-2-3-???", raise_exception=True) + mock_redis_delete.assert_has_calls([expected_call, expected_call]) def test_doesnt_update_api_if_redis_delete_by_pattern_fails(mocker, mocked_redis_client, cache): From 61e0cf0aa589f98ad61ebb1a85926d730ad61834 Mon Sep 17 00:00:00 2001 From: David McDonald Date: Wed, 22 May 2024 14:59:23 +0100 Subject: [PATCH 038/211] Bump version to 77.2.1 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 149540c53..70a3fc6f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 77.2.1 + +* Change redis delete behaviour to error, rather than end up with stale data, if Redis is unavailable. + ## 77.2.0 * `NotifyTask`: include pid and other structured fields in completion log messages diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 0b9dcb1fd..aa908f7ce 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "77.2.0" # dbe1d856722af4ad0303161298e2aba3 +__version__ = "77.2.1" # 096378d793de4f0d83045dda5032c767 From 919f39c6f5a3c383650d4ebc4cc8f5ee5ea80876 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 8 May 2024 10:21:07 +0100 Subject: [PATCH 039/211] move postal_address.py to new recipient_validation folder first in a series of commits to move phone number and email validation in to one centralised location too --- notifications_utils/recipient_validation/__init__.py | 0 .../{ => recipient_validation}/postal_address.py | 0 notifications_utils/recipients.py | 4 ++-- notifications_utils/template.py | 2 +- tests/{ => recipient_validation}/test_postal_address.py | 8 ++++---- 5 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 notifications_utils/recipient_validation/__init__.py rename notifications_utils/{ => recipient_validation}/postal_address.py (100%) rename tests/{ => recipient_validation}/test_postal_address.py (99%) diff --git a/notifications_utils/recipient_validation/__init__.py b/notifications_utils/recipient_validation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/notifications_utils/postal_address.py b/notifications_utils/recipient_validation/postal_address.py similarity index 100% rename from notifications_utils/postal_address.py rename to notifications_utils/recipient_validation/postal_address.py diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index d8b0aa413..ff91f30d9 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -22,7 +22,7 @@ COUNTRY_PREFIXES, INTERNATIONAL_BILLING_RATES, ) -from notifications_utils.postal_address import ( +from notifications_utils.recipient_validation.postal_address import ( address_line_7_key, address_lines_1_to_6_and_postcode_keys, address_lines_1_to_7_keys, @@ -430,7 +430,7 @@ def recipient(self): @property def as_postal_address(self): - from notifications_utils.postal_address import PostalAddress + from notifications_utils.recipient_validation.postal_address import PostalAddress return PostalAddress.from_personalisation( self.recipient_and_personalisation, diff --git a/notifications_utils/template.py b/notifications_utils/template.py index 57da5a783..41197dbc6 100644 --- a/notifications_utils/template.py +++ b/notifications_utils/template.py @@ -44,8 +44,8 @@ notify_letter_qrcode_validator, notify_plain_text_email_markdown, ) -from notifications_utils.postal_address import PostalAddress, address_lines_1_to_7_keys from notifications_utils.qr_code import QrCodeTooLong +from notifications_utils.recipient_validation.postal_address import PostalAddress, address_lines_1_to_7_keys from notifications_utils.sanitise_text import SanitiseSMS from notifications_utils.take import Take from notifications_utils.template_change import TemplateChange diff --git a/tests/test_postal_address.py b/tests/recipient_validation/test_postal_address.py similarity index 99% rename from tests/test_postal_address.py rename to tests/recipient_validation/test_postal_address.py index 6949a6783..6daadfbdc 100644 --- a/tests/test_postal_address.py +++ b/tests/recipient_validation/test_postal_address.py @@ -3,7 +3,7 @@ from notifications_utils.countries import Country from notifications_utils.countries.data import Postage from notifications_utils.insensitive_dict import InsensitiveDict -from notifications_utils.postal_address import ( +from notifications_utils.recipient_validation.postal_address import ( PostalAddress, _is_a_real_uk_postcode, format_postcode_for_printing, @@ -630,7 +630,7 @@ def test_normalise_postcode(postcode, normalised_postcode): ("N5 1AA", True), ("SO14 6WB", True), ("so14 6wb", True), - ("so14\u00A06wb", True), + ("so14\u00a06wb", True), # invalida / incomplete postcodes ("N5", False), ("SO144 6WB", False), @@ -661,7 +661,7 @@ def test_if_postcode_is_a_real_uk_postcode(postcode, result): def test_if_postcode_is_a_real_uk_postcode_normalises_before_checking_postcode(mocker): - normalise_postcode_mock = mocker.patch("notifications_utils.postal_address.normalise_postcode") + normalise_postcode_mock = mocker.patch("notifications_utils.recipient_validation.postal_address.normalise_postcode") normalise_postcode_mock.return_value = "SW11AA" assert _is_a_real_uk_postcode("sw1 1aa") is True @@ -675,7 +675,7 @@ def test_if_postcode_is_a_real_uk_postcode_normalises_before_checking_postcode(m ("N5 3EF", "N5 3EF"), ("N53EF ", "N5 3EF"), ("n53Ef", "N5 3EF"), - ("n5 \u00A0 \t 3Ef", "N5 3EF"), + ("n5 \u00a0 \t 3Ef", "N5 3EF"), ("SO146WB", "SO14 6WB"), ("GIR0AA", "GIR 0AA"), ("BF11AA", "BF1 1AA"), From c6265d81c45db338a89b7b9eee1ca2b544e12159 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 8 May 2024 10:25:19 +0100 Subject: [PATCH 040/211] move recipient validation errors to their own file also have a new shared base class so we don't have phone number errors extending from email address errors. This was raised six (!) years ago but never sorted out[^1]. [^1]: https://github.com/alphagov/notifications-api/pull/1590/files#diff-463068b4d0340d64d95bb9d1e816b1d4da643fd30341c637f2745fff3609559bR46-R47 --- .../recipient_validation/errors.py | 17 +++++++++++++++++ notifications_utils/recipients.py | 16 ++-------------- tests/test_recipient_validation.py | 3 +-- 3 files changed, 20 insertions(+), 16 deletions(-) create mode 100644 notifications_utils/recipient_validation/errors.py diff --git a/notifications_utils/recipient_validation/errors.py b/notifications_utils/recipient_validation/errors.py new file mode 100644 index 000000000..0b9cb12ff --- /dev/null +++ b/notifications_utils/recipient_validation/errors.py @@ -0,0 +1,17 @@ +class InvalidRecipientError(Exception): + message = "Not a valid recipient address" + + def __init__(self, message=None): + super().__init__(message or self.message) + + +class InvalidEmailError(InvalidRecipientError): + message = "Not a valid email address" + + +class InvalidPhoneError(InvalidRecipientError): + message = "Not a valid phone number" + + +class InvalidAddressError(InvalidRecipientError): + message = "Not a valid postal address" diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index ff91f30d9..1f0bfd624 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -22,6 +22,7 @@ COUNTRY_PREFIXES, INTERNATIONAL_BILLING_RATES, ) +from notifications_utils.recipient_validation.errors import InvalidEmailError, InvalidPhoneError, InvalidRecipientError from notifications_utils.recipient_validation.postal_address import ( address_line_7_key, address_lines_1_to_6_and_postcode_keys, @@ -330,7 +331,7 @@ def _get_error_for_field(self, key, value): # noqa: C901 validate_email_address(value) if self.template_type == "sms": validate_phone_number(value, international=self.allow_international_sms) - except (InvalidEmailError, InvalidPhoneError) as error: + except InvalidRecipientError as error: return str(error) if InsensitiveDict.make_key(key) not in self.placeholders_as_column_keys: @@ -470,19 +471,6 @@ def recipient_error(self): return self.error not in {None, self.missing_field_error} -class InvalidEmailError(Exception): - def __init__(self, message=None): - super().__init__(message or "Not a valid email address") - - -class InvalidPhoneError(InvalidEmailError): - pass - - -class InvalidAddressError(InvalidEmailError): - pass - - def normalise_phone_number(number): for character in ALL_WHITESPACE + "()-+": number = number.replace(character, "") diff --git a/tests/test_recipient_validation.py b/tests/test_recipient_validation.py index 346addd8b..d90023183 100644 --- a/tests/test_recipient_validation.py +++ b/tests/test_recipient_validation.py @@ -1,8 +1,7 @@ import pytest +from notifications_utils.recipient_validation.errors import InvalidEmailError, InvalidPhoneError from notifications_utils.recipients import ( - InvalidEmailError, - InvalidPhoneError, allowed_to_send_to, format_phone_number_human_readable, format_recipient, From 0de634ba208449d931734a8684ec25d17b881fdf Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 8 May 2024 10:57:19 +0100 Subject: [PATCH 041/211] move phone number validation to its own file --- .../recipient_validation/phone_number.py | 155 ++++++++ notifications_utils/recipients.py | 154 +------- .../recipient_validation/test_phone_number.py | 374 ++++++++++++++++++ tests/test_international_billing_rates.py | 2 +- tests/test_recipient_validation.py | 352 +---------------- 5 files changed, 536 insertions(+), 501 deletions(-) create mode 100644 notifications_utils/recipient_validation/phone_number.py create mode 100644 tests/recipient_validation/test_phone_number.py diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py new file mode 100644 index 000000000..4354dcc25 --- /dev/null +++ b/notifications_utils/recipient_validation/phone_number.py @@ -0,0 +1,155 @@ +from collections import namedtuple + +import phonenumbers +from flask import current_app + +from notifications_utils.formatters import ( + ALL_WHITESPACE, +) +from notifications_utils.international_billing_rates import ( + COUNTRY_PREFIXES, + INTERNATIONAL_BILLING_RATES, +) +from notifications_utils.recipient_validation.errors import InvalidPhoneError + +UK_PREFIX = "44" + + +international_phone_info = namedtuple( + "PhoneNumber", + [ + "international", + "crown_dependency", + "country_prefix", + "billable_units", + ], +) + + +def normalise_phone_number(number): + for character in ALL_WHITESPACE + "()-+": + number = number.replace(character, "") + + try: + list(map(int, number)) + except ValueError as e: + raise InvalidPhoneError("Mobile numbers can only include: 0 1 2 3 4 5 6 7 8 9 ( ) + -") from e + + return number.lstrip("0") + + +def is_uk_phone_number(number): + if number.startswith("0") and not number.startswith("00"): + return True + + number = normalise_phone_number(number) + + if number.startswith(UK_PREFIX) or (number.startswith("7") and len(number) < 11): + return True + + return False + + +def get_international_phone_info(number): + number = validate_phone_number(number, international=True) + prefix = get_international_prefix(number) + crown_dependency = _is_a_crown_dependency_number(number) + + return international_phone_info( + international=(prefix != UK_PREFIX or crown_dependency), + crown_dependency=crown_dependency, + country_prefix=prefix, + billable_units=get_billable_units_for_prefix(prefix), + ) + + +CROWN_DEPENDENCY_RANGES = ["7781", "7839", "7911", "7509", "7797", "7937", "7700", "7829", "7624", "7524", "7924"] + + +def _is_a_crown_dependency_number(number): + num_in_crown_dependency_range = number[2:6] in CROWN_DEPENDENCY_RANGES + num_in_tv_range = number[2:9] == "7700900" + + return num_in_crown_dependency_range and not num_in_tv_range + + +def get_international_prefix(number): + return next((prefix for prefix in COUNTRY_PREFIXES if number.startswith(prefix)), None) + + +def get_billable_units_for_prefix(prefix): + return INTERNATIONAL_BILLING_RATES[prefix]["billable_units"] + + +def use_numeric_sender(number): + prefix = get_international_prefix(normalise_phone_number(number)) + return INTERNATIONAL_BILLING_RATES[prefix]["attributes"]["alpha"] == "NO" + + +def validate_uk_phone_number(number): + number = normalise_phone_number(number).lstrip(UK_PREFIX).lstrip("0") + + if not number.startswith("7"): + raise InvalidPhoneError( + "This does not look like a UK mobile number – double check the mobile number you entered" + ) + + if len(number) > 10: + raise InvalidPhoneError("Mobile number is too long") + + if len(number) < 10: + raise InvalidPhoneError("Mobile number is too short") + + return f"{UK_PREFIX}{number}" + + +def validate_phone_number(number, international=False): + if (not international) or is_uk_phone_number(number): + return validate_uk_phone_number(number) + + number = normalise_phone_number(number) + + if len(number) < 8: + raise InvalidPhoneError("Mobile number is too short") + + if len(number) > 15: + raise InvalidPhoneError("Mobile number is too long") + + if get_international_prefix(number) is None: + raise InvalidPhoneError("Country code not found - double check the mobile number you entered") + + return number + + +validate_and_format_phone_number = validate_phone_number + + +def try_validate_and_format_phone_number(number, international=None, log_msg=None): + """ + For use in places where you shouldn't error if the phone number is invalid - for example if firetext pass us + something in + """ + try: + return validate_and_format_phone_number(number, international) + except InvalidPhoneError as exc: + if log_msg: + current_app.logger.warning("%s: %s", log_msg, exc) + return number + + +def format_phone_number_human_readable(phone_number): + try: + phone_number = validate_phone_number(phone_number, international=True) + except InvalidPhoneError: + # if there was a validation error, we want to shortcut out here, but still display the number on the front end + return phone_number + international_phone_info = get_international_phone_info(phone_number) + + return phonenumbers.format_number( + phonenumbers.parse("+" + phone_number, None), + ( + phonenumbers.PhoneNumberFormat.INTERNATIONAL + if international_phone_info.international + else phonenumbers.PhoneNumberFormat.NATIONAL + ), + ) diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index 1f0bfd624..a0fb699b8 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -1,28 +1,24 @@ import csv import re import sys -from collections import namedtuple from contextlib import suppress from functools import lru_cache from io import StringIO from itertools import islice from typing import Optional, cast -import phonenumbers -from flask import current_app from ordered_set import OrderedSet from notifications_utils.formatters import ( - ALL_WHITESPACE, strip_all_whitespace, strip_and_remove_obscure_whitespace, ) from notifications_utils.insensitive_dict import InsensitiveDict -from notifications_utils.international_billing_rates import ( - COUNTRY_PREFIXES, - INTERNATIONAL_BILLING_RATES, -) from notifications_utils.recipient_validation.errors import InvalidEmailError, InvalidPhoneError, InvalidRecipientError +from notifications_utils.recipient_validation.phone_number import ( + validate_and_format_phone_number, + validate_phone_number, +) from notifications_utils.recipient_validation.postal_address import ( address_line_7_key, address_lines_1_to_6_and_postcode_keys, @@ -33,8 +29,6 @@ from . import EMAIL_REGEX_PATTERN, hostname_part, tld_part from .qr_code import QrCodeTooLong -uk_prefix = "44" - first_column_headings = { "email": ["email address"], "sms": ["phone number"], @@ -471,128 +465,6 @@ def recipient_error(self): return self.error not in {None, self.missing_field_error} -def normalise_phone_number(number): - for character in ALL_WHITESPACE + "()-+": - number = number.replace(character, "") - - try: - list(map(int, number)) - except ValueError as e: - raise InvalidPhoneError("Mobile numbers can only include: 0 1 2 3 4 5 6 7 8 9 ( ) + -") from e - - return number.lstrip("0") - - -def is_uk_phone_number(number): - if number.startswith("0") and not number.startswith("00"): - return True - - number = normalise_phone_number(number) - - if number.startswith(uk_prefix) or (number.startswith("7") and len(number) < 11): - return True - - return False - - -international_phone_info = namedtuple( - "PhoneNumber", - [ - "international", - "crown_dependency", - "country_prefix", - "billable_units", - ], -) - - -def get_international_phone_info(number): - number = validate_phone_number(number, international=True) - prefix = get_international_prefix(number) - crown_dependency = _is_a_crown_dependency_number(number) - - return international_phone_info( - international=(prefix != uk_prefix or crown_dependency), - crown_dependency=crown_dependency, - country_prefix=prefix, - billable_units=get_billable_units_for_prefix(prefix), - ) - - -CROWN_DEPENDENCY_RANGES = ["7781", "7839", "7911", "7509", "7797", "7937", "7700", "7829", "7624", "7524", "7924"] - - -def _is_a_crown_dependency_number(number): - num_in_crown_dependency_range = number[2:6] in CROWN_DEPENDENCY_RANGES - num_in_tv_range = number[2:9] == "7700900" - - return num_in_crown_dependency_range and not num_in_tv_range - - -def get_international_prefix(number): - return next((prefix for prefix in COUNTRY_PREFIXES if number.startswith(prefix)), None) - - -def get_billable_units_for_prefix(prefix): - return INTERNATIONAL_BILLING_RATES[prefix]["billable_units"] - - -def use_numeric_sender(number): - prefix = get_international_prefix(normalise_phone_number(number)) - return INTERNATIONAL_BILLING_RATES[prefix]["attributes"]["alpha"] == "NO" - - -def validate_uk_phone_number(number): - number = normalise_phone_number(number).lstrip(uk_prefix).lstrip("0") - - if not number.startswith("7"): - raise InvalidPhoneError( - "This does not look like a UK mobile number – double check the mobile number you entered" - ) - - if len(number) > 10: - raise InvalidPhoneError("Mobile number is too long") - - if len(number) < 10: - raise InvalidPhoneError("Mobile number is too short") - - return f"{uk_prefix}{number}" - - -def validate_phone_number(number, international=False): - if (not international) or is_uk_phone_number(number): - return validate_uk_phone_number(number) - - number = normalise_phone_number(number) - - if len(number) < 8: - raise InvalidPhoneError("Mobile number is too short") - - if len(number) > 15: - raise InvalidPhoneError("Mobile number is too long") - - if get_international_prefix(number) is None: - raise InvalidPhoneError("Country code not found - double check the mobile number you entered") - - return number - - -validate_and_format_phone_number = validate_phone_number - - -def try_validate_and_format_phone_number(number, international=None, log_msg=None): - """ - For use in places where you shouldn't error if the phone number is invalid - for example if firetext pass us - something in - """ - try: - return validate_and_format_phone_number(number, international) - except InvalidPhoneError as exc: - if log_msg: - current_app.logger.warning("%s: %s", log_msg, exc) - return number - - def validate_email_address(email_address): # noqa (C901 too complex) # almost exactly the same as by https://github.com/wtforms/wtforms/blob/master/wtforms/validators.py, # with minor tweaks for SES compatibility - to avoid complications we are a lot stricter with the local part @@ -655,24 +527,6 @@ def format_recipient(recipient): return recipient -def format_phone_number_human_readable(phone_number): - try: - phone_number = validate_phone_number(phone_number, international=True) - except InvalidPhoneError: - # if there was a validation error, we want to shortcut out here, but still display the number on the front end - return phone_number - international_phone_info = get_international_phone_info(phone_number) - - return phonenumbers.format_number( - phonenumbers.parse("+" + phone_number, None), - ( - phonenumbers.PhoneNumberFormat.INTERNATIONAL - if international_phone_info.international - else phonenumbers.PhoneNumberFormat.NATIONAL - ), - ) - - def allowed_to_send_to(recipient, allowlist): return format_recipient(recipient) in {format_recipient(x) for x in allowlist} diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py new file mode 100644 index 000000000..d067a5224 --- /dev/null +++ b/tests/recipient_validation/test_phone_number.py @@ -0,0 +1,374 @@ +import pytest + +from notifications_utils.recipient_validation.errors import InvalidPhoneError +from notifications_utils.recipient_validation.phone_number import ( + format_phone_number_human_readable, + get_international_phone_info, + international_phone_info, + is_uk_phone_number, + normalise_phone_number, + try_validate_and_format_phone_number, + validate_and_format_phone_number, + validate_phone_number, +) +from notifications_utils.recipients import ( + allowed_to_send_to, + format_recipient, +) + +valid_uk_phone_numbers = [ + "7123456789", + "07123456789", + "07123 456789", + "07123-456-789", + "00447123456789", + "00 44 7123456789", + "+447123456789", + "+44 7123 456 789", + "+44 (0)7123 456 789", + "\u200b\t\t+44 (0)7123 \ufeff 456 789 \r\n", +] + + +valid_international_phone_numbers = [ + "71234567890", # Russia + "1-202-555-0104", # USA + "+12025550104", # USA + "0012025550104", # USA + "+0012025550104", # USA + "23051234567", # Mauritius, + "+682 12345", # Cook islands + "+3312345678", + "003312345678", + "1-2345-12345-12345", # 15 digits +] + + +valid_phone_numbers = valid_uk_phone_numbers + valid_international_phone_numbers + + +invalid_uk_phone_numbers = sum( + [ + [(phone_number, error) for phone_number in group] + for error, group in [ + ( + "Mobile number is too long", + ( + "712345678910", + "0712345678910", + "0044712345678910", + "0044712345678910", + "+44 (0)7123 456 789 10", + ), + ), + ( + "Mobile number is too short", + ( + "0712345678", + "004471234567", + "00447123456", + "+44 (0)7123 456 78", + ), + ), + ( + "This does not look like a UK mobile number – double check the mobile number you entered", + ( + "08081 570364", + "+44 8081 570364", + "0117 496 0860", + "+44 117 496 0860", + "020 7946 0991", + "+44 20 7946 0991", + ), + ), + ( + "Mobile numbers can only include: 0 1 2 3 4 5 6 7 8 9 ( ) + -", + ( + "07890x32109", + "07123 456789...", + "07123 ☟☜⬇⬆☞☝", + "07123☟☜⬇⬆☞☝", + '07";DROP TABLE;"', + "+44 07ab cde fgh", + "ALPHANUM3R1C", + ), + ), + ] + ], + [], +) + + +invalid_phone_numbers = list( + filter( + lambda number: number[0] + not in { + "712345678910", # Could be Russia + }, + invalid_uk_phone_numbers, + ) +) + [ + ("800000000000", "Country code not found - double check the mobile number you entered"), + ("1234567", "Mobile number is too short"), + ("+682 1234", "Mobile number is too short"), # Cook Islands phone numbers can be 5 digits + ("+12345 12345 12345 6", "Mobile number is too long"), +] + + +@pytest.mark.parametrize("phone_number", valid_international_phone_numbers) +def test_detect_international_phone_numbers(phone_number): + assert is_uk_phone_number(phone_number) is False + + +@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) +def test_detect_uk_phone_numbers(phone_number): + assert is_uk_phone_number(phone_number) is True + + +@pytest.mark.parametrize( + "phone_number, expected_info", + [ + ( + "07900900123", + international_phone_info( + international=False, + crown_dependency=False, + country_prefix="44", # UK + billable_units=1, + ), + ), + ( + "07700900123", + international_phone_info( + international=False, + crown_dependency=False, + country_prefix="44", # Number in TV range + billable_units=1, + ), + ), + ( + "07700800123", + international_phone_info( + international=True, + crown_dependency=True, + country_prefix="44", # UK Crown dependency, so prefix same as UK + billable_units=1, + ), + ), + ( + "20-12-1234-1234", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="20", # Egypt + billable_units=3, + ), + ), + ( + "00201212341234", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="20", # Egypt + billable_units=3, + ), + ), + ( + "1664000000000", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="1664", # Montserrat + billable_units=3, + ), + ), + ( + "71234567890", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="7", # Russia + billable_units=4, + ), + ), + ( + "1-202-555-0104", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="1", # USA + billable_units=1, + ), + ), + ( + "+23051234567", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="230", # Mauritius + billable_units=2, + ), + ), + ], +) +def test_get_international_info(phone_number, expected_info): + assert get_international_phone_info(phone_number) == expected_info + + +@pytest.mark.parametrize( + "phone_number", + [ + "abcd", + "079OO900123", + pytest.param("", marks=pytest.mark.xfail), + pytest.param("12345", marks=pytest.mark.xfail), + pytest.param("+12345", marks=pytest.mark.xfail), + pytest.param("1-2-3-4-5", marks=pytest.mark.xfail), + pytest.param("1 2 3 4 5", marks=pytest.mark.xfail), + pytest.param("(1)2345", marks=pytest.mark.xfail), + ], +) +def test_normalise_phone_number_raises_if_unparseable_characters(phone_number): + with pytest.raises(InvalidPhoneError): + normalise_phone_number(phone_number) + + +@pytest.mark.parametrize( + "phone_number", + [ + "+21 4321 0987", + "00997 1234 7890", + "801234-7890", + "(8-0)-1234-7890", + ], +) +def test_get_international_info_raises(phone_number): + with pytest.raises(InvalidPhoneError) as error: + get_international_phone_info(phone_number) + assert str(error.value) == "Country code not found - double check the mobile number you entered" + + +@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) +@pytest.mark.parametrize( + "extra_args", + [ + {}, + {"international": False}, + ], +) +def test_phone_number_accepts_valid_values(extra_args, phone_number): + try: + validate_phone_number(phone_number, **extra_args) + except InvalidPhoneError: + pytest.fail("Unexpected InvalidPhoneError") + + +@pytest.mark.parametrize("phone_number", valid_phone_numbers) +def test_phone_number_accepts_valid_international_values(phone_number): + try: + validate_phone_number(phone_number, international=True) + except InvalidPhoneError: + pytest.fail("Unexpected InvalidPhoneError") + + +@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) +def test_valid_uk_phone_number_can_be_formatted_consistently(phone_number): + assert validate_and_format_phone_number(phone_number) == "447123456789" + + +@pytest.mark.parametrize( + "phone_number, expected_formatted", + [ + ("71234567890", "71234567890"), + ("1-202-555-0104", "12025550104"), + ("+12025550104", "12025550104"), + ("0012025550104", "12025550104"), + ("+0012025550104", "12025550104"), + ("23051234567", "23051234567"), + ], +) +def test_valid_international_phone_number_can_be_formatted_consistently(phone_number, expected_formatted): + assert validate_and_format_phone_number(phone_number, international=True) == expected_formatted + + +@pytest.mark.parametrize("phone_number, error_message", invalid_uk_phone_numbers) +@pytest.mark.parametrize( + "extra_args", + [ + {}, + {"international": False}, + ], +) +def test_phone_number_rejects_invalid_values(extra_args, phone_number, error_message): + with pytest.raises(InvalidPhoneError) as e: + validate_phone_number(phone_number, **extra_args) + assert error_message == str(e.value) + + +@pytest.mark.parametrize("phone_number, error_message", invalid_phone_numbers) +def test_phone_number_rejects_invalid_international_values(phone_number, error_message): + with pytest.raises(InvalidPhoneError) as e: + validate_phone_number(phone_number, international=True) + assert error_message == str(e.value) + + +@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) +def test_validates_against_guestlist_of_phone_numbers(phone_number): + assert allowed_to_send_to(phone_number, ["07123456789", "07700900460", "test@example.com"]) + assert not allowed_to_send_to(phone_number, ["07700900460", "07700900461", "test@example.com"]) + + +@pytest.mark.parametrize( + "recipient_number, allowlist_number", + [ + ["1-202-555-0104", "0012025550104"], + ["0012025550104", "1-202-555-0104"], + ], +) +def test_validates_against_guestlist_of_international_phone_numbers(recipient_number, allowlist_number): + assert allowed_to_send_to(recipient_number, [allowlist_number]) + + +@pytest.mark.parametrize( + "phone_number, expected_formatted", + [ + ("07900900123", "07900 900123"), # UK + ("+44(0)7900900123", "07900 900123"), # UK + ("447900900123", "07900 900123"), # UK + ("20-12-1234-1234", "+20 12 12341234"), # Egypt + ("00201212341234", "+20 12 12341234"), # Egypt + ("1664 0000000", "+1 664-000-0000"), # Montserrat + ("7 499 1231212", "+7 499 123-12-12"), # Moscow (Russia) + ("1-202-555-0104", "+1 202-555-0104"), # Washington DC (USA) + ("+23051234567", "+230 5123 4567"), # Mauritius + ("33(0)1 12345678", "+33 1 12 34 56 78"), # Paris (France) + ], +) +def test_format_uk_and_international_phone_numbers(phone_number, expected_formatted): + assert format_phone_number_human_readable(phone_number) == expected_formatted + + +@pytest.mark.parametrize( + "recipient, expected_formatted", + [ + (True, ""), + (False, ""), + (0, ""), + (0.1, ""), + (None, ""), + ("foo", "foo"), + ("TeSt@ExAmPl3.com", "test@exampl3.com"), + ("+4407900 900 123", "447900900123"), + ("+1 800 555 5555", "18005555555"), + ], +) +def test_format_recipient(recipient, expected_formatted): + assert format_recipient(recipient) == expected_formatted + + +def test_try_format_recipient_doesnt_throw(): + assert try_validate_and_format_phone_number("ALPHANUM3R1C") == "ALPHANUM3R1C" + + +def test_format_phone_number_human_readable_doenst_throw(): + assert format_phone_number_human_readable("ALPHANUM3R1C") == "ALPHANUM3R1C" diff --git a/tests/test_international_billing_rates.py b/tests/test_international_billing_rates.py index 24645b94d..e5df93fc8 100644 --- a/tests/test_international_billing_rates.py +++ b/tests/test_international_billing_rates.py @@ -4,7 +4,7 @@ COUNTRY_PREFIXES, INTERNATIONAL_BILLING_RATES, ) -from notifications_utils.recipients import use_numeric_sender +from notifications_utils.recipient_validation.phone_number import use_numeric_sender def test_international_billing_rates_exists(): diff --git a/tests/test_recipient_validation.py b/tests/test_recipient_validation.py index d90023183..0d769e13c 100644 --- a/tests/test_recipient_validation.py +++ b/tests/test_recipient_validation.py @@ -1,119 +1,12 @@ import pytest -from notifications_utils.recipient_validation.errors import InvalidEmailError, InvalidPhoneError +from notifications_utils.recipient_validation.errors import InvalidEmailError from notifications_utils.recipients import ( allowed_to_send_to, - format_phone_number_human_readable, format_recipient, - get_international_phone_info, - international_phone_info, - is_uk_phone_number, - normalise_phone_number, - try_validate_and_format_phone_number, - validate_and_format_phone_number, validate_email_address, - validate_phone_number, ) -valid_uk_phone_numbers = [ - "7123456789", - "07123456789", - "07123 456789", - "07123-456-789", - "00447123456789", - "00 44 7123456789", - "+447123456789", - "+44 7123 456 789", - "+44 (0)7123 456 789", - "\u200B\t\t+44 (0)7123 \uFEFF 456 789 \r\n", -] - - -valid_international_phone_numbers = [ - "71234567890", # Russia - "1-202-555-0104", # USA - "+12025550104", # USA - "0012025550104", # USA - "+0012025550104", # USA - "23051234567", # Mauritius, - "+682 12345", # Cook islands - "+3312345678", - "003312345678", - "1-2345-12345-12345", # 15 digits -] - - -valid_phone_numbers = valid_uk_phone_numbers + valid_international_phone_numbers - - -invalid_uk_phone_numbers = sum( - [ - [(phone_number, error) for phone_number in group] - for error, group in [ - ( - "Mobile number is too long", - ( - "712345678910", - "0712345678910", - "0044712345678910", - "0044712345678910", - "+44 (0)7123 456 789 10", - ), - ), - ( - "Mobile number is too short", - ( - "0712345678", - "004471234567", - "00447123456", - "+44 (0)7123 456 78", - ), - ), - ( - "This does not look like a UK mobile number – double check the mobile number you entered", - ( - "08081 570364", - "+44 8081 570364", - "0117 496 0860", - "+44 117 496 0860", - "020 7946 0991", - "+44 20 7946 0991", - ), - ), - ( - "Mobile numbers can only include: 0 1 2 3 4 5 6 7 8 9 ( ) + -", - ( - "07890x32109", - "07123 456789...", - "07123 ☟☜⬇⬆☞☝", - "07123☟☜⬇⬆☞☝", - '07";DROP TABLE;"', - "+44 07ab cde fgh", - "ALPHANUM3R1C", - ), - ), - ] - ], - [], -) - - -invalid_phone_numbers = list( - filter( - lambda number: number[0] - not in { - "712345678910", # Could be Russia - }, - invalid_uk_phone_numbers, - ) -) + [ - ("800000000000", "Country code not found - double check the mobile number you entered"), - ("1234567", "Mobile number is too short"), - ("+682 1234", "Mobile number is too short"), # Cook Islands phone numbers can be 5 digits - ("+12345 12345 12345 6", "Mobile number is too long"), -] - - valid_email_addresses = ( "email@domain.com", "email@domain.COM", @@ -167,203 +60,6 @@ ) -@pytest.mark.parametrize("phone_number", valid_international_phone_numbers) -def test_detect_international_phone_numbers(phone_number): - assert is_uk_phone_number(phone_number) is False - - -@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) -def test_detect_uk_phone_numbers(phone_number): - assert is_uk_phone_number(phone_number) is True - - -@pytest.mark.parametrize( - "phone_number, expected_info", - [ - ( - "07900900123", - international_phone_info( - international=False, - crown_dependency=False, - country_prefix="44", # UK - billable_units=1, - ), - ), - ( - "07700900123", - international_phone_info( - international=False, - crown_dependency=False, - country_prefix="44", # Number in TV range - billable_units=1, - ), - ), - ( - "07700800123", - international_phone_info( - international=True, - crown_dependency=True, - country_prefix="44", # UK Crown dependency, so prefix same as UK - billable_units=1, - ), - ), - ( - "20-12-1234-1234", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="20", # Egypt - billable_units=3, - ), - ), - ( - "00201212341234", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="20", # Egypt - billable_units=3, - ), - ), - ( - "1664000000000", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="1664", # Montserrat - billable_units=3, - ), - ), - ( - "71234567890", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="7", # Russia - billable_units=4, - ), - ), - ( - "1-202-555-0104", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="1", # USA - billable_units=1, - ), - ), - ( - "+23051234567", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="230", # Mauritius - billable_units=2, - ), - ), - ], -) -def test_get_international_info(phone_number, expected_info): - assert get_international_phone_info(phone_number) == expected_info - - -@pytest.mark.parametrize( - "phone_number", - [ - "abcd", - "079OO900123", - pytest.param("", marks=pytest.mark.xfail), - pytest.param("12345", marks=pytest.mark.xfail), - pytest.param("+12345", marks=pytest.mark.xfail), - pytest.param("1-2-3-4-5", marks=pytest.mark.xfail), - pytest.param("1 2 3 4 5", marks=pytest.mark.xfail), - pytest.param("(1)2345", marks=pytest.mark.xfail), - ], -) -def test_normalise_phone_number_raises_if_unparseable_characters(phone_number): - with pytest.raises(InvalidPhoneError): - normalise_phone_number(phone_number) - - -@pytest.mark.parametrize( - "phone_number", - [ - "+21 4321 0987", - "00997 1234 7890", - "801234-7890", - "(8-0)-1234-7890", - ], -) -def test_get_international_info_raises(phone_number): - with pytest.raises(InvalidPhoneError) as error: - get_international_phone_info(phone_number) - assert str(error.value) == "Country code not found - double check the mobile number you entered" - - -@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) -@pytest.mark.parametrize( - "extra_args", - [ - {}, - {"international": False}, - ], -) -def test_phone_number_accepts_valid_values(extra_args, phone_number): - try: - validate_phone_number(phone_number, **extra_args) - except InvalidPhoneError: - pytest.fail("Unexpected InvalidPhoneError") - - -@pytest.mark.parametrize("phone_number", valid_phone_numbers) -def test_phone_number_accepts_valid_international_values(phone_number): - try: - validate_phone_number(phone_number, international=True) - except InvalidPhoneError: - pytest.fail("Unexpected InvalidPhoneError") - - -@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) -def test_valid_uk_phone_number_can_be_formatted_consistently(phone_number): - assert validate_and_format_phone_number(phone_number) == "447123456789" - - -@pytest.mark.parametrize( - "phone_number, expected_formatted", - [ - ("71234567890", "71234567890"), - ("1-202-555-0104", "12025550104"), - ("+12025550104", "12025550104"), - ("0012025550104", "12025550104"), - ("+0012025550104", "12025550104"), - ("23051234567", "23051234567"), - ], -) -def test_valid_international_phone_number_can_be_formatted_consistently(phone_number, expected_formatted): - assert validate_and_format_phone_number(phone_number, international=True) == expected_formatted - - -@pytest.mark.parametrize("phone_number, error_message", invalid_uk_phone_numbers) -@pytest.mark.parametrize( - "extra_args", - [ - {}, - {"international": False}, - ], -) -def test_phone_number_rejects_invalid_values(extra_args, phone_number, error_message): - with pytest.raises(InvalidPhoneError) as e: - validate_phone_number(phone_number, **extra_args) - assert error_message == str(e.value) - - -@pytest.mark.parametrize("phone_number, error_message", invalid_phone_numbers) -def test_phone_number_rejects_invalid_international_values(phone_number, error_message): - with pytest.raises(InvalidPhoneError) as e: - validate_phone_number(phone_number, international=True) - assert error_message == str(e.value) - - @pytest.mark.parametrize("email_address", valid_email_addresses) def test_validate_email_address_accepts_valid(email_address): try: @@ -378,7 +74,7 @@ def test_validate_email_address_accepts_valid(email_address): " email@domain.com ", "\temail@domain.com", "\temail@domain.com\n", - "\u200Bemail@domain.com\u200B", + "\u200bemail@domain.com\u200b", ], ) def test_validate_email_address_strips_whitespace(email): @@ -392,47 +88,11 @@ def test_validate_email_address_raises_for_invalid(email_address): assert str(e.value) == "Not a valid email address" -@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) -def test_validates_against_guestlist_of_phone_numbers(phone_number): - assert allowed_to_send_to(phone_number, ["07123456789", "07700900460", "test@example.com"]) - assert not allowed_to_send_to(phone_number, ["07700900460", "07700900461", "test@example.com"]) - - -@pytest.mark.parametrize( - "recipient_number, allowlist_number", - [ - ["1-202-555-0104", "0012025550104"], - ["0012025550104", "1-202-555-0104"], - ], -) -def test_validates_against_guestlist_of_international_phone_numbers(recipient_number, allowlist_number): - assert allowed_to_send_to(recipient_number, [allowlist_number]) - - @pytest.mark.parametrize("email_address", valid_email_addresses) def test_validates_against_guestlist_of_email_addresses(email_address): assert not allowed_to_send_to(email_address, ["very_special_and_unique@example.com"]) -@pytest.mark.parametrize( - "phone_number, expected_formatted", - [ - ("07900900123", "07900 900123"), # UK - ("+44(0)7900900123", "07900 900123"), # UK - ("447900900123", "07900 900123"), # UK - ("20-12-1234-1234", "+20 12 12341234"), # Egypt - ("00201212341234", "+20 12 12341234"), # Egypt - ("1664 0000000", "+1 664-000-0000"), # Montserrat - ("7 499 1231212", "+7 499 123-12-12"), # Moscow (Russia) - ("1-202-555-0104", "+1 202-555-0104"), # Washington DC (USA) - ("+23051234567", "+230 5123 4567"), # Mauritius - ("33(0)1 12345678", "+33 1 12 34 56 78"), # Paris (France) - ], -) -def test_format_uk_and_international_phone_numbers(phone_number, expected_formatted): - assert format_phone_number_human_readable(phone_number) == expected_formatted - - @pytest.mark.parametrize( "recipient, expected_formatted", [ @@ -449,11 +109,3 @@ def test_format_uk_and_international_phone_numbers(phone_number, expected_format ) def test_format_recipient(recipient, expected_formatted): assert format_recipient(recipient) == expected_formatted - - -def test_try_format_recipient_doesnt_throw(): - assert try_validate_and_format_phone_number("ALPHANUM3R1C") == "ALPHANUM3R1C" - - -def test_format_phone_number_human_readable_doenst_throw(): - assert format_phone_number_human_readable("ALPHANUM3R1C") == "ALPHANUM3R1C" From d17f29550479fd2f3d043569d4912b91de50ac02 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 8 May 2024 11:11:50 +0100 Subject: [PATCH 042/211] move email validation to its own file note that i moved the email-specific regex constants out of notifications_utils/__init__.py and into the email_address file itself. these are only used by the email_address code itself and in one place in document download, and now that they're in a smaller file they seem reasonable to place here --- notifications_utils/__init__.py | 9 -- .../recipient_validation/email_address.py | 66 +++++++++++++ notifications_utils/recipients.py | 57 +----------- .../test_email_address.py | 90 ++++++++++++++++++ tests/test_recipient_validation.py | 92 +------------------ 5 files changed, 161 insertions(+), 153 deletions(-) create mode 100644 notifications_utils/recipient_validation/email_address.py create mode 100644 tests/recipient_validation/test_email_address.py diff --git a/notifications_utils/__init__.py b/notifications_utils/__init__.py index 58658ee49..c0cfa18e0 100644 --- a/notifications_utils/__init__.py +++ b/notifications_utils/__init__.py @@ -3,15 +3,6 @@ SMS_CHAR_COUNT_LIMIT = 918 # 153 * 6, no network issues but check with providers before upping this further LETTER_MAX_PAGE_COUNT = 10 -# regexes for use in recipients.validate_email_address. -# Valid characters taken from https://en.wikipedia.org/wiki/Email_address#Local-part -# Note: Normal apostrophe eg `Firstname-o'surname@domain.com` is allowed. -# hostname_part regex: xn in regex signifies possible punycode conversions, which would start `xn--`; -# the hyphens are matched for later in the regex. -hostname_part = re.compile(r"^(xn|[a-z0-9]+)(-?-[a-z0-9]+)*$", re.IGNORECASE) -tld_part = re.compile(r"^([a-z]{2,63}|xn--([a-z0-9]+-)*[a-z0-9]+)$", re.IGNORECASE) -VALID_LOCAL_CHARS = r"a-zA-Z0-9.!#$%&'*+/=?^_`{|}~\-" -EMAIL_REGEX_PATTERN = r"^[{}]+@([^.@][^@\s]+)$".format(VALID_LOCAL_CHARS) email_with_smart_quotes_regex = re.compile( # matches wider than an email - everything between an at sign and the nearest whitespace r"(^|\s)\S+@\S+(\s|$)", diff --git a/notifications_utils/recipient_validation/email_address.py b/notifications_utils/recipient_validation/email_address.py new file mode 100644 index 000000000..c17055286 --- /dev/null +++ b/notifications_utils/recipient_validation/email_address.py @@ -0,0 +1,66 @@ +import re + +from notifications_utils.formatters import ( + strip_and_remove_obscure_whitespace, +) +from notifications_utils.recipient_validation.errors import InvalidEmailError + +# Valid characters taken from https://en.wikipedia.org/wiki/Email_address#Local-part +# Note: Normal apostrophe eg `Firstname-o'surname@domain.com` is allowed. +# hostname_part regex: xn in regex signifies possible punycode conversions, which would start `xn--`; +# the hyphens are matched for later in the regex. +hostname_part = re.compile(r"^(xn|[a-z0-9]+)(-?-[a-z0-9]+)*$", re.IGNORECASE) +tld_part = re.compile(r"^([a-z]{2,63}|xn--([a-z0-9]+-)*[a-z0-9]+)$", re.IGNORECASE) +VALID_LOCAL_CHARS = r"a-zA-Z0-9.!#$%&'*+/=?^_`{|}~\-" +EMAIL_REGEX_PATTERN = r"^[{}]+@([^.@][^@\s]+)$".format(VALID_LOCAL_CHARS) + + +def validate_email_address(email_address): # noqa (C901 too complex) + # almost exactly the same as by https://github.com/wtforms/wtforms/blob/master/wtforms/validators.py, + # with minor tweaks for SES compatibility - to avoid complications we are a lot stricter with the local part + # than neccessary - we don't allow any double quotes or semicolons to prevent SES Technical Failures + email_address = strip_and_remove_obscure_whitespace(email_address) + match = re.match(EMAIL_REGEX_PATTERN, email_address) + + # not an email + if not match: + raise InvalidEmailError + + if len(email_address) > 320: + raise InvalidEmailError + + # don't allow consecutive periods in either part + if ".." in email_address: + raise InvalidEmailError + + hostname = match.group(1) + + # idna = "Internationalized domain name" - this encode/decode cycle converts unicode into its accurate ascii + # representation as the web uses. '例え.テスト'.encode('idna') == b'xn--r8jz45g.xn--zckzah' + try: + hostname = hostname.encode("idna").decode("ascii") + except UnicodeError as e: + raise InvalidEmailError from e + + parts = hostname.split(".") + + if len(hostname) > 253 or len(parts) < 2: + raise InvalidEmailError + + for part in parts: + if not part or len(part) > 63 or not hostname_part.match(part): + raise InvalidEmailError + + # if the part after the last . is not a valid TLD then bail out + if not tld_part.match(parts[-1]): + raise InvalidEmailError + + return email_address + + +def format_email_address(email_address): + return strip_and_remove_obscure_whitespace(email_address.lower()) + + +def validate_and_format_email_address(email_address): + return format_email_address(validate_email_address(email_address)) diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index a0fb699b8..ef47aee05 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -1,5 +1,4 @@ import csv -import re import sys from contextlib import suppress from functools import lru_cache @@ -14,6 +13,10 @@ strip_and_remove_obscure_whitespace, ) from notifications_utils.insensitive_dict import InsensitiveDict +from notifications_utils.recipient_validation.email_address import ( + validate_and_format_email_address, + validate_email_address, +) from notifications_utils.recipient_validation.errors import InvalidEmailError, InvalidPhoneError, InvalidRecipientError from notifications_utils.recipient_validation.phone_number import ( validate_and_format_phone_number, @@ -26,7 +29,6 @@ ) from notifications_utils.template import BaseLetterTemplate, Template -from . import EMAIL_REGEX_PATTERN, hostname_part, tld_part from .qr_code import QrCodeTooLong first_column_headings = { @@ -465,57 +467,6 @@ def recipient_error(self): return self.error not in {None, self.missing_field_error} -def validate_email_address(email_address): # noqa (C901 too complex) - # almost exactly the same as by https://github.com/wtforms/wtforms/blob/master/wtforms/validators.py, - # with minor tweaks for SES compatibility - to avoid complications we are a lot stricter with the local part - # than neccessary - we don't allow any double quotes or semicolons to prevent SES Technical Failures - email_address = strip_and_remove_obscure_whitespace(email_address) - match = re.match(EMAIL_REGEX_PATTERN, email_address) - - # not an email - if not match: - raise InvalidEmailError - - if len(email_address) > 320: - raise InvalidEmailError - - # don't allow consecutive periods in either part - if ".." in email_address: - raise InvalidEmailError - - hostname = match.group(1) - - # idna = "Internationalized domain name" - this encode/decode cycle converts unicode into its accurate ascii - # representation as the web uses. '例え.テスト'.encode('idna') == b'xn--r8jz45g.xn--zckzah' - try: - hostname = hostname.encode("idna").decode("ascii") - except UnicodeError as e: - raise InvalidEmailError from e - - parts = hostname.split(".") - - if len(hostname) > 253 or len(parts) < 2: - raise InvalidEmailError - - for part in parts: - if not part or len(part) > 63 or not hostname_part.match(part): - raise InvalidEmailError - - # if the part after the last . is not a valid TLD then bail out - if not tld_part.match(parts[-1]): - raise InvalidEmailError - - return email_address - - -def format_email_address(email_address): - return strip_and_remove_obscure_whitespace(email_address.lower()) - - -def validate_and_format_email_address(email_address): - return format_email_address(validate_email_address(email_address)) - - @lru_cache(maxsize=32, typed=False) def format_recipient(recipient): if not isinstance(recipient, str): diff --git a/tests/recipient_validation/test_email_address.py b/tests/recipient_validation/test_email_address.py new file mode 100644 index 000000000..03ff0e2cc --- /dev/null +++ b/tests/recipient_validation/test_email_address.py @@ -0,0 +1,90 @@ +import pytest + +from notifications_utils.recipient_validation.email_address import validate_email_address +from notifications_utils.recipient_validation.errors import InvalidEmailError +from notifications_utils.recipients import allowed_to_send_to + +valid_email_addresses = ( + "email@domain.com", + "email@domain.COM", + "firstname.lastname@domain.com", + "firstname.o'lastname@domain.com", + "email@subdomain.domain.com", + "firstname+lastname@domain.com", + "1234567890@domain.com", + "email@domain-one.com", + "_______@domain.com", + "email@domain.name", + "email@domain.superlongtld", + "email@domain.co.jp", + "firstname-lastname@domain.com", + "info@german-financial-services.vermögensberatung", + "info@german-financial-services.reallylongarbitrarytldthatiswaytoohugejustincase", + "japanese-info@例え.テスト", + "email@double--hyphen.com", +) +invalid_email_addresses = ( + "email@123.123.123.123", + "email@[123.123.123.123]", + "plainaddress", + "@no-local-part.com", + "Outlook Contact ", + "no-at.domain.com", + "no-tld@domain", + ";beginning-semicolon@domain.co.uk", + "middle-semicolon@domain.co;uk", + "trailing-semicolon@domain.com;", + '"email+leading-quotes@domain.com', + 'email+middle"-quotes@domain.com', + '"quoted-local-part"@domain.com', + '"quoted@domain.com"', + "lots-of-dots@domain..gov..uk", + "two-dots..in-local@domain.com", + "multiple@domains@domain.com", + "spaces in local@domain.com", + "spaces-in-domain@dom ain.com", + "underscores-in-domain@dom_ain.com", + "pipe-in-domain@example.com|gov.uk", + "comma,in-local@gov.uk", + "comma-in-domain@domain,gov.uk", + "pound-sign-in-local£@domain.com", + "local-with-’-apostrophe@domain.com", + "local-with-”-quotes@domain.com", + "domain-starts-with-a-dot@.domain.com", + "brackets(in)local@domain.com", + f"email-too-long-{'a' * 320}@example.com", + "incorrect-punycode@xn---something.com", +) + + +@pytest.mark.parametrize("email_address", valid_email_addresses) +def test_validate_email_address_accepts_valid(email_address): + try: + assert validate_email_address(email_address) == email_address + except InvalidEmailError: + pytest.fail("Unexpected InvalidEmailError") + + +@pytest.mark.parametrize( + "email", + [ + " email@domain.com ", + "\temail@domain.com", + "\temail@domain.com\n", + "\u200bemail@domain.com\u200b", + ], +) +def test_validate_email_address_strips_whitespace(email): + assert validate_email_address(email) == "email@domain.com" + + +@pytest.mark.parametrize("email_address", invalid_email_addresses) +def test_validate_email_address_raises_for_invalid(email_address): + with pytest.raises(InvalidEmailError) as e: + validate_email_address(email_address) + assert str(e.value) == "Not a valid email address" + + +@pytest.mark.parametrize("email_address", valid_email_addresses) +def test_validates_against_guestlist_of_email_addresses(email_address): + assert not allowed_to_send_to(email_address, ["very_special_and_unique@example.com"]) diff --git a/tests/test_recipient_validation.py b/tests/test_recipient_validation.py index 0d769e13c..1cdd2a64e 100644 --- a/tests/test_recipient_validation.py +++ b/tests/test_recipient_validation.py @@ -1,96 +1,6 @@ import pytest -from notifications_utils.recipient_validation.errors import InvalidEmailError -from notifications_utils.recipients import ( - allowed_to_send_to, - format_recipient, - validate_email_address, -) - -valid_email_addresses = ( - "email@domain.com", - "email@domain.COM", - "firstname.lastname@domain.com", - "firstname.o'lastname@domain.com", - "email@subdomain.domain.com", - "firstname+lastname@domain.com", - "1234567890@domain.com", - "email@domain-one.com", - "_______@domain.com", - "email@domain.name", - "email@domain.superlongtld", - "email@domain.co.jp", - "firstname-lastname@domain.com", - "info@german-financial-services.vermögensberatung", - "info@german-financial-services.reallylongarbitrarytldthatiswaytoohugejustincase", - "japanese-info@例え.テスト", - "email@double--hyphen.com", -) -invalid_email_addresses = ( - "email@123.123.123.123", - "email@[123.123.123.123]", - "plainaddress", - "@no-local-part.com", - "Outlook Contact ", - "no-at.domain.com", - "no-tld@domain", - ";beginning-semicolon@domain.co.uk", - "middle-semicolon@domain.co;uk", - "trailing-semicolon@domain.com;", - '"email+leading-quotes@domain.com', - 'email+middle"-quotes@domain.com', - '"quoted-local-part"@domain.com', - '"quoted@domain.com"', - "lots-of-dots@domain..gov..uk", - "two-dots..in-local@domain.com", - "multiple@domains@domain.com", - "spaces in local@domain.com", - "spaces-in-domain@dom ain.com", - "underscores-in-domain@dom_ain.com", - "pipe-in-domain@example.com|gov.uk", - "comma,in-local@gov.uk", - "comma-in-domain@domain,gov.uk", - "pound-sign-in-local£@domain.com", - "local-with-’-apostrophe@domain.com", - "local-with-”-quotes@domain.com", - "domain-starts-with-a-dot@.domain.com", - "brackets(in)local@domain.com", - f"email-too-long-{'a' * 320}@example.com", - "incorrect-punycode@xn---something.com", -) - - -@pytest.mark.parametrize("email_address", valid_email_addresses) -def test_validate_email_address_accepts_valid(email_address): - try: - assert validate_email_address(email_address) == email_address - except InvalidEmailError: - pytest.fail("Unexpected InvalidEmailError") - - -@pytest.mark.parametrize( - "email", - [ - " email@domain.com ", - "\temail@domain.com", - "\temail@domain.com\n", - "\u200bemail@domain.com\u200b", - ], -) -def test_validate_email_address_strips_whitespace(email): - assert validate_email_address(email) == "email@domain.com" - - -@pytest.mark.parametrize("email_address", invalid_email_addresses) -def test_validate_email_address_raises_for_invalid(email_address): - with pytest.raises(InvalidEmailError) as e: - validate_email_address(email_address) - assert str(e.value) == "Not a valid email address" - - -@pytest.mark.parametrize("email_address", valid_email_addresses) -def test_validates_against_guestlist_of_email_addresses(email_address): - assert not allowed_to_send_to(email_address, ["very_special_and_unique@example.com"]) +from notifications_utils.recipients import format_recipient @pytest.mark.parametrize( From dfc45535b08b550814d7d423c092ec23a9db4783 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 8 May 2024 11:14:24 +0100 Subject: [PATCH 043/211] change imports in recipient.py mostly so that in places that use recipient.py, we'll be forced to update the imports so that they're using the new paths to keep everywhere up to date --- notifications_utils/recipients.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index ef47aee05..8ece2a69c 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -13,15 +13,8 @@ strip_and_remove_obscure_whitespace, ) from notifications_utils.insensitive_dict import InsensitiveDict -from notifications_utils.recipient_validation.email_address import ( - validate_and_format_email_address, - validate_email_address, -) +from notifications_utils.recipient_validation import email_address, phone_number from notifications_utils.recipient_validation.errors import InvalidEmailError, InvalidPhoneError, InvalidRecipientError -from notifications_utils.recipient_validation.phone_number import ( - validate_and_format_phone_number, - validate_phone_number, -) from notifications_utils.recipient_validation.postal_address import ( address_line_7_key, address_lines_1_to_6_and_postcode_keys, @@ -324,9 +317,9 @@ def _get_error_for_field(self, key, value): # noqa: C901 return Cell.missing_field_error try: if self.template_type == "email": - validate_email_address(value) + email_address.validate_email_address(value) if self.template_type == "sms": - validate_phone_number(value, international=self.allow_international_sms) + phone_number.validate_phone_number(value, international=self.allow_international_sms) except InvalidRecipientError as error: return str(error) @@ -472,9 +465,9 @@ def format_recipient(recipient): if not isinstance(recipient, str): return "" with suppress(InvalidPhoneError): - return validate_and_format_phone_number(recipient, international=True) + return phone_number.validate_and_format_phone_number(recipient, international=True) with suppress(InvalidEmailError): - return validate_and_format_email_address(recipient) + return email_address.validate_and_format_email_address(recipient) return recipient From 869b244c505e91443a9649247a12d67134a3abab Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 8 May 2024 11:18:13 +0100 Subject: [PATCH 044/211] changelog and major version bump --- CHANGELOG.md | 9 +++++++++ notifications_utils/version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70a3fc6f5..e32a026c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # CHANGELOG +## 78.0.0 + +* BREAKING CHANGE: recipient validation code has all been moved into separate files in a shared folder. Functionality is unchanged. + - Email address validation can be found in `notifications_utils.recipient_validation.email_address` + - Phone number validation can be found in `notifications_utils.recipient_validation.phone_number` + - Postal address validation can be found in `notifications_utils.recipient_validation.postal_address` +* BREAKING CHANGE: InvalidPhoneError and InvalidAddressError no longer extend InvalidEmailError. + - if you wish to handle all recipient validation errors, please use `notifications_utils.recipient_validation.errors.InvalidRecipientError` + ## 77.2.1 * Change redis delete behaviour to error, rather than end up with stale data, if Redis is unavailable. diff --git a/notifications_utils/version.py b/notifications_utils/version.py index aa908f7ce..912140c60 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "77.2.1" # 096378d793de4f0d83045dda5032c767 +__version__ = "78.0.0" # ef32afe465129c9213c29a3883eb54ef From bceaaf83724c7cbb534791571a749b511b24b478 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 11 Jun 2024 11:18:04 +0100 Subject: [PATCH 045/211] Restrict postcodes to valid UK postcode zones MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The postcode zone is the first 1, 2 or 3 letters of the postcode. There are about 120 postcode zones defined in the UK. At the moment we allow postcodes to start with any 1 or 2 letters. This means we allow plenty of postcode which aren’t real, for example `NF1 1AA`. By restricting the first n letters of the postcode to only valid postcode zones we can reduce the number of letters which DVLA manually flag to us as having invalid addresses. *** List taken from: http://www.ons.gov.uk/ons/guide-method/geography/products/postcode-directories/-nspp-/onspd-user-guide-and-version-notes.zip --- .../recipient_validation/postal_address.py | 131 +++++++++++++++++- .../test_postal_address.py | 1 + 2 files changed, 131 insertions(+), 1 deletion(-) diff --git a/notifications_utils/recipient_validation/postal_address.py b/notifications_utils/recipient_validation/postal_address.py index 231d7c33e..c21aae2eb 100644 --- a/notifications_utils/recipient_validation/postal_address.py +++ b/notifications_utils/recipient_validation/postal_address.py @@ -22,6 +22,135 @@ address_line_7_key = "address_line_7" address_lines_1_to_7_keys = address_lines_1_to_6_keys + [address_line_7_key] country_UK = Country(UK) +uk_postcode_zones = "|".join({ + "AB", + "AL", + "B", + "BA", + "BB", + "BD", + "BF", + "BH", + "BL", + "BN", + "BR", + "BS", + "BT", + "BX", + "CA", + "CB", + "CF", + "CH", + "CM", + "CO", + "CR", + "CT", + "CV", + "CW", + "DA", + "DD", + "DE", + "DG", + "DH", + "DL", + "DN", + "DT", + "DY", + "E", + "EC", + "EH", + "EN", + "EX", + "FK", + "FY", + "G", + "GL", + "GU", + "GY", + "HA", + "HD", + "HG", + "HP", + "HR", + "HS", + "HU", + "HX", + "IG", + "IM", + "IP", + "IV", + "JE", + "KA", + "KT", + "KW", + "KY", + "L", + "GIR", + "LA", + "LD", + "LE", + "LL", + "LN", + "LS", + "LU", + "M", + "ME", + "MK", + "ML", + "N", + "NE", + "NG", + "NN", + "NP", + "NR", + "NW", + "OL", + "OX", + "PA", + "PE", + "PH", + "PL", + "PO", + "PR", + "RG", + "RH", + "RM", + "S", + "SA", + "SE", + "SG", + "SK", + "SL", + "SM", + "SN", + "SO", + "SP", + "SR", + "SS", + "ST", + "SW", + "SY", + "TA", + "TD", + "TF", + "TN", + "TQ", + "TR", + "TS", + "TW", + "UB", + "W", + "WA", + "WC", + "WD", + "WF", + "WN", + "WR", + "WS", + "WV", + "YO", + "ZE", +}) class PostalAddress: @@ -232,7 +361,7 @@ def _is_a_real_uk_postcode(postcode): if normalised == "GX111AA": return False # GIR0AA is Girobank - pattern = re.compile(r"([A-Z]{1,2}[0-9][0-9A-Z]?[0-9][A-BD-HJLNP-UW-Z]{2})|(GIR0AA)") + pattern = re.compile(rf"(({uk_postcode_zones})[0-9][0-9A-Z]?[0-9][A-BD-HJLNP-UW-Z]{{2}})|(GIR0AA)") return bool(pattern.fullmatch(normalised)) diff --git a/tests/recipient_validation/test_postal_address.py b/tests/recipient_validation/test_postal_address.py index 6daadfbdc..d6a0fd258 100644 --- a/tests/recipient_validation/test_postal_address.py +++ b/tests/recipient_validation/test_postal_address.py @@ -635,6 +635,7 @@ def test_normalise_postcode(postcode, normalised_postcode): ("N5", False), ("SO144 6WB", False), ("SO14 6WBA", False), + ("NF1 1AA", False), ("", False), ("Bad postcode", False), # British Forces Post Office numbers are not postcodes From e5d7ae7d79fd3193c05508220e5419f3da5e5480 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 11 Jun 2024 11:33:21 +0100 Subject: [PATCH 046/211] Refactor into data file This saves us clogging up `postal_address.py` with a long list --- .../countries/_data/uk-postcode-zones.txt | 127 +++++++++++++++++ notifications_utils/countries/data.py | 1 + .../recipient_validation/postal_address.py | 133 +----------------- 3 files changed, 130 insertions(+), 131 deletions(-) create mode 100644 notifications_utils/countries/_data/uk-postcode-zones.txt diff --git a/notifications_utils/countries/_data/uk-postcode-zones.txt b/notifications_utils/countries/_data/uk-postcode-zones.txt new file mode 100644 index 000000000..f1574d5eb --- /dev/null +++ b/notifications_utils/countries/_data/uk-postcode-zones.txt @@ -0,0 +1,127 @@ +AB +AL +B +BA +BB +BD +BF +BH +BL +BN +BR +BS +BT +BX +CA +CB +CF +CH +CM +CO +CR +CT +CV +CW +DA +DD +DE +DG +DH +DL +DN +DT +DY +E +EC +EH +EN +EX +FK +FY +G +GL +GU +GY +HA +HD +HG +HP +HR +HS +HU +HX +IG +IM +IP +IV +JE +KA +KT +KW +KY +L +GIR +LA +LD +LE +LL +LN +LS +LU +M +ME +MK +ML +N +NE +NG +NN +NP +NR +NW +OL +OX +PA +PE +PH +PL +PO +PR +RG +RH +RM +S +SA +SE +SG +SK +SL +SM +SN +SO +SP +SR +SS +ST +SW +SY +TA +TD +TF +TN +TQ +TR +TS +TW +UB +W +WA +WC +WD +WF +WN +WR +WS +WV +YO +ZE \ No newline at end of file diff --git a/notifications_utils/countries/data.py b/notifications_utils/countries/data.py index 0acabb22d..46d56c9ca 100644 --- a/notifications_utils/countries/data.py +++ b/notifications_utils/countries/data.py @@ -32,6 +32,7 @@ def find_canonical(item, graph, key): WELSH_NAMES = list(_load_data("welsh-names.json").items()) _UK_ISLANDS_LIST = _load_data("uk-islands.txt") _EUROPEAN_ISLANDS_LIST = _load_data("european-islands.txt") +UK_POSTCODE_ZONES = _load_data("uk-postcode-zones.txt") CURRENT_AND_ENDED_COUNTRIES_AND_TERRITORIES = [ find_canonical(item, _graph, item["names"]["en-GB"]) for item in _graph.values() diff --git a/notifications_utils/recipient_validation/postal_address.py b/notifications_utils/recipient_validation/postal_address.py index c21aae2eb..756c0ee17 100644 --- a/notifications_utils/recipient_validation/postal_address.py +++ b/notifications_utils/recipient_validation/postal_address.py @@ -2,7 +2,7 @@ from functools import lru_cache from notifications_utils.countries import UK, Country, CountryNotFoundError -from notifications_utils.countries.data import Postage +from notifications_utils.countries.data import UK_POSTCODE_ZONES, Postage from notifications_utils.formatters import ( get_lines_with_normalised_whitespace, remove_whitespace, @@ -22,135 +22,6 @@ address_line_7_key = "address_line_7" address_lines_1_to_7_keys = address_lines_1_to_6_keys + [address_line_7_key] country_UK = Country(UK) -uk_postcode_zones = "|".join({ - "AB", - "AL", - "B", - "BA", - "BB", - "BD", - "BF", - "BH", - "BL", - "BN", - "BR", - "BS", - "BT", - "BX", - "CA", - "CB", - "CF", - "CH", - "CM", - "CO", - "CR", - "CT", - "CV", - "CW", - "DA", - "DD", - "DE", - "DG", - "DH", - "DL", - "DN", - "DT", - "DY", - "E", - "EC", - "EH", - "EN", - "EX", - "FK", - "FY", - "G", - "GL", - "GU", - "GY", - "HA", - "HD", - "HG", - "HP", - "HR", - "HS", - "HU", - "HX", - "IG", - "IM", - "IP", - "IV", - "JE", - "KA", - "KT", - "KW", - "KY", - "L", - "GIR", - "LA", - "LD", - "LE", - "LL", - "LN", - "LS", - "LU", - "M", - "ME", - "MK", - "ML", - "N", - "NE", - "NG", - "NN", - "NP", - "NR", - "NW", - "OL", - "OX", - "PA", - "PE", - "PH", - "PL", - "PO", - "PR", - "RG", - "RH", - "RM", - "S", - "SA", - "SE", - "SG", - "SK", - "SL", - "SM", - "SN", - "SO", - "SP", - "SR", - "SS", - "ST", - "SW", - "SY", - "TA", - "TD", - "TF", - "TN", - "TQ", - "TR", - "TS", - "TW", - "UB", - "W", - "WA", - "WC", - "WD", - "WF", - "WN", - "WR", - "WS", - "WV", - "YO", - "ZE", -}) class PostalAddress: @@ -361,7 +232,7 @@ def _is_a_real_uk_postcode(postcode): if normalised == "GX111AA": return False # GIR0AA is Girobank - pattern = re.compile(rf"(({uk_postcode_zones})[0-9][0-9A-Z]?[0-9][A-BD-HJLNP-UW-Z]{{2}})|(GIR0AA)") + pattern = re.compile(rf"(({'|'.join(UK_POSTCODE_ZONES)})[0-9][0-9A-Z]?[0-9][A-BD-HJLNP-UW-Z]{{2}})|(GIR0AA)") return bool(pattern.fullmatch(normalised)) From 9cf59782ec445a71e954397ec5e6b9386b4be4b1 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 11 Jun 2024 11:18:59 +0100 Subject: [PATCH 047/211] Remove special case for Gibraltar `GX` is not a valid UK postcode zone so we will reject it anyway, without having to treat it as a special case. --- notifications_utils/recipient_validation/postal_address.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/notifications_utils/recipient_validation/postal_address.py b/notifications_utils/recipient_validation/postal_address.py index 756c0ee17..59c990228 100644 --- a/notifications_utils/recipient_validation/postal_address.py +++ b/notifications_utils/recipient_validation/postal_address.py @@ -229,8 +229,6 @@ def normalise_postcode(postcode): def _is_a_real_uk_postcode(postcode): normalised = normalise_postcode(postcode) - if normalised == "GX111AA": - return False # GIR0AA is Girobank pattern = re.compile(rf"(({'|'.join(UK_POSTCODE_ZONES)})[0-9][0-9A-Z]?[0-9][A-BD-HJLNP-UW-Z]{{2}})|(GIR0AA)") return bool(pattern.fullmatch(normalised)) From dc68a333a52ea893101427d92def0b04ccfde3f7 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 11 Jun 2024 11:23:49 +0100 Subject: [PATCH 048/211] Minor version bump to 78.1.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e32a026c6..bf8782461 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 78.1.0 + +* Restrict postcodes to valid UK postcode zones + ## 78.0.0 * BREAKING CHANGE: recipient validation code has all been moved into separate files in a shared folder. Functionality is unchanged. diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 912140c60..5d1cea430 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "78.0.0" # ef32afe465129c9213c29a3883eb54ef +__version__ = "78.1.0" # 772a0feff4a65c5402deb31eff79b8d1 From 47cfcdfd5c2068a3e5b73d22f8b5f1f726035c82 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 10 Jun 2024 14:40:18 +0100 Subject: [PATCH 049/211] bump minimum versions of subdependencies Many libraries we're only using much more recent versions than utils has pinned anyway (like flask or pypdf), so it's easy to bring them up to the oldest version in use without any risk. Others have outstanding CVEs (like jinja) so it's good to make sure we always have secure versions. Others like boto3 we've already assessed the changelogs elsewhere[^1] Many major changes were just dropping support for old versions of python Some like cachetools needed a bit of checking as they changed import structure (but we're not affected in this instance) [^1]: https://github.com/alphagov/notifications-api/pull/4082 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- setup.py | 24 ++++++++++++------------ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf8782461..de399bddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 78.2.0 + +* Bumped minimum versions of select subdependencies + ## 78.1.0 * Restrict postcodes to valid UK postcode zones diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 5d1cea430..c45f721c8 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "78.1.0" # 772a0feff4a65c5402deb31eff79b8d1 +__version__ = "78.2.0" # 47dcea615e142c6d524dde2064804fbd diff --git a/setup.py b/setup.py index 8c10b6ab5..29db929fb 100644 --- a/setup.py +++ b/setup.py @@ -23,24 +23,24 @@ packages=find_packages(exclude=("tests",)), include_package_data=True, install_requires=[ - "cachetools>=4.1.1", + "cachetools>=5.3.3", "mistune<2.0.0", # v2 is totally incompatible with unclear benefit - "requests>=2.25.0", - "python-json-logger>=2.0.1", - "Flask>=2.1.1", + "requests>=2.32.0", + "python-json-logger>=2.0.7", + "Flask>=3.0.0", "gunicorn>=20.0.0", "ordered-set>=4.1.0", - "Jinja2>=2.11.3", - "statsd>=3.3.0", + "Jinja2>=3.1.4", + "statsd>=4.0.1", "Flask-Redis>=0.4.0", - "pyyaml>=5.3.1", + "pyyaml>=6.0.1", "phonenumbers>=8.13.18", - "pytz>=2020.4", + "pytz>=2024.1", "smartypants>=2.0.1", - "pypdf>=3.9.0", - "itsdangerous>=1.1.0", - "govuk-bank-holidays>=0.10,<1.0", - "boto3>=1.19.4", + "pypdf>=3.13.0", + "itsdangerous>=2.1.2", + "govuk-bank-holidays>=0.14", + "boto3>=1.34.100", "segno>=1.5.2,<2.0.0", ], ) From 4eb96e3ca5bb4dcb096aba4eccb089bb64514c56 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 10 Jun 2024 14:58:25 +0100 Subject: [PATCH 050/211] add 2026 bank holidays --- notifications_utils/data/bank-holidays.json | 162 ++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/notifications_utils/data/bank-holidays.json b/notifications_utils/data/bank-holidays.json index 4c8f0adb6..cd08d8bb3 100644 --- a/notifications_utils/data/bank-holidays.json +++ b/notifications_utils/data/bank-holidays.json @@ -403,6 +403,54 @@ "date": "2025-12-26", "notes": "", "bunting": true + }, + { + "title": "New Year’s Day", + "date": "2026-01-01", + "notes": "", + "bunting": true + }, + { + "title": "Good Friday", + "date": "2026-04-03", + "notes": "", + "bunting": false + }, + { + "title": "Easter Monday", + "date": "2026-04-06", + "notes": "", + "bunting": true + }, + { + "title": "Early May bank holiday", + "date": "2026-05-04", + "notes": "", + "bunting": true + }, + { + "title": "Spring bank holiday", + "date": "2026-05-25", + "notes": "", + "bunting": true + }, + { + "title": "Summer bank holiday", + "date": "2026-08-31", + "notes": "", + "bunting": true + }, + { + "title": "Christmas Day", + "date": "2026-12-25", + "notes": "", + "bunting": true + }, + { + "title": "Boxing Day", + "date": "2026-12-28", + "notes": "Substitute day", + "bunting": true } ] }, @@ -858,6 +906,60 @@ "date": "2025-12-26", "notes": "", "bunting": true + }, + { + "title": "New Year’s Day", + "date": "2026-01-01", + "notes": "", + "bunting": true + }, + { + "title": "2nd January", + "date": "2026-01-02", + "notes": "", + "bunting": true + }, + { + "title": "Good Friday", + "date": "2026-04-03", + "notes": "", + "bunting": false + }, + { + "title": "Early May bank holiday", + "date": "2026-05-04", + "notes": "", + "bunting": true + }, + { + "title": "Spring bank holiday", + "date": "2026-05-25", + "notes": "", + "bunting": true + }, + { + "title": "Summer bank holiday", + "date": "2026-08-03", + "notes": "", + "bunting": true + }, + { + "title": "St Andrew’s Day", + "date": "2026-11-30", + "notes": "", + "bunting": true + }, + { + "title": "Christmas Day", + "date": "2026-12-25", + "notes": "", + "bunting": true + }, + { + "title": "Boxing Day", + "date": "2026-12-28", + "notes": "Substitute day", + "bunting": true } ] }, @@ -1361,6 +1463,66 @@ "date": "2025-12-26", "notes": "", "bunting": true + }, + { + "title": "New Year’s Day", + "date": "2026-01-01", + "notes": "", + "bunting": true + }, + { + "title": "St Patrick’s Day", + "date": "2026-03-17", + "notes": "", + "bunting": true + }, + { + "title": "Good Friday", + "date": "2026-04-03", + "notes": "", + "bunting": false + }, + { + "title": "Easter Monday", + "date": "2026-04-06", + "notes": "", + "bunting": true + }, + { + "title": "Early May bank holiday", + "date": "2026-05-04", + "notes": "", + "bunting": true + }, + { + "title": "Spring bank holiday", + "date": "2026-05-25", + "notes": "", + "bunting": true + }, + { + "title": "Battle of the Boyne (Orangemen’s Day)", + "date": "2026-07-13", + "notes": "Substitute day", + "bunting": false + }, + { + "title": "Summer bank holiday", + "date": "2026-08-31", + "notes": "", + "bunting": true + }, + { + "title": "Christmas Day", + "date": "2026-12-25", + "notes": "", + "bunting": true + }, + { + "title": "Boxing Day", + "date": "2026-12-28", + "notes": "Substitute day", + "bunting": true } ] } From 26c86b6a8814b0f5948d9f1ea271d67141dbf50e Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 8 May 2024 16:48:51 +0100 Subject: [PATCH 051/211] Enable pyupgrade linting rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pyupgrade[1] automatically upgrades syntax for newer versions of Python. Newer syntax is often friendlier and less verbose – for example f-string formatting rather than calls to `str.format`. It has a total of 42 rules[2]. This commit adds it to our linter config, and then fixes all the instances of old syntax it found (the vaast majority of these were automatic by running Ruff with the `--fix` flag). 1. https://pypi.org/project/pyupgrade/ 2. https://docs.astral.sh/ruff/rules/#pyupgrade-up --- .../clients/redis/redis_client.py | 33 +++++++++---------- .../clients/zendesk/zendesk_client.py | 9 +++-- notifications_utils/logging/__init__.py | 2 +- notifications_utils/markdown.py | 8 ++--- .../recipient_validation/email_address.py | 2 +- notifications_utils/recipients.py | 14 ++++---- notifications_utils/request_helper.py | 2 +- notifications_utils/safe_string.py | 2 +- notifications_utils/template.py | 4 +-- notifications_utils/testing/comparisons.py | 4 +-- pyproject.toml | 1 + tests/test_recipient_csv.py | 28 ++++++---------- tests/test_serialised_model.py | 9 ++--- 13 files changed, 50 insertions(+), 68 deletions(-) diff --git a/notifications_utils/clients/redis/redis_client.py b/notifications_utils/clients/redis/redis_client.py index d940bf74d..24736737f 100644 --- a/notifications_utils/clients/redis/redis_client.py +++ b/notifications_utils/clients/redis/redis_client.py @@ -2,13 +2,16 @@ import uuid from time import time from types import TracebackType -from typing import Optional, Type + +# (`Type` is deprecated in favour of `type` but we need to match the +# signature of the method we are stubbing) +from typing import Type # noqa: UP035 from flask import current_app from flask_redis import FlaskRedis # expose redis exceptions so that they can be caught -from redis.exceptions import RedisError # noqa +from redis.exceptions import RedisError # noqa: F401 from redis.lock import Lock from redis.typing import Number @@ -26,15 +29,11 @@ def prepare_value(val): # things redis-py natively supports if isinstance( val, - ( - bytes, - str, - numbers.Number, - ), + bytes | str | numbers.Number, ): return val # things we know we can safely cast to string - elif isinstance(val, (uuid.UUID,)): + elif isinstance(val, uuid.UUID): return str(val) else: raise ValueError(f"cannot cast {type(val)} to a string") @@ -195,10 +194,10 @@ def __init__( self, redis, name: str, - timeout: Optional[Number] = None, + timeout: Number | None = None, sleep: Number = 0.1, blocking: bool = True, - blocking_timeout: Optional[Number] = None, + blocking_timeout: Number | None = None, thread_local: bool = True, ): self._locked = False @@ -210,18 +209,18 @@ def __enter__(self) -> "StubLock": def __exit__( self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], + exc_type: Type[BaseException] | None, # noqa: UP006 + exc_value: BaseException | None, + traceback: TracebackType | None, ) -> None: self._locked = False def acquire( self, - sleep: Optional[Number] = None, - blocking: Optional[bool] = None, - blocking_timeout: Optional[Number] = None, - token: Optional[str] = None, + sleep: Number | None = None, + blocking: bool | None = None, + blocking_timeout: Number | None = None, + token: str | None = None, ) -> bool: self._locked = True return True diff --git a/notifications_utils/clients/zendesk/zendesk_client.py b/notifications_utils/clients/zendesk/zendesk_client.py index f7774cb73..43ad531bd 100644 --- a/notifications_utils/clients/zendesk/zendesk_client.py +++ b/notifications_utils/clients/zendesk/zendesk_client.py @@ -2,7 +2,6 @@ import datetime import enum import typing -from typing import Optional from urllib.parse import urlencode import requests @@ -112,9 +111,9 @@ def _upload_attachment(self, attachment: NotifySupportTicketAttachment): def update_ticket( self, ticket_id: int, - comment: Optional[NotifySupportTicketComment] = None, - due_at: Optional[datetime.datetime] = None, - status: Optional[NotifySupportTicketStatus] = None, + comment: NotifySupportTicketComment | None = None, + due_at: datetime.datetime | None = None, + status: NotifySupportTicketStatus | None = None, ): data = {"ticket": {}} @@ -182,7 +181,7 @@ def __init__( user_name=None, user_email=None, requester_sees_message_content=True, - notify_ticket_type: Optional[NotifyTicketType] = None, + notify_ticket_type: NotifyTicketType | None = None, notify_task_type=None, org_id=None, org_type=None, diff --git a/notifications_utils/logging/__init__.py b/notifications_utils/logging/__init__.py index ec9a08c5f..813d4bbe9 100644 --- a/notifications_utils/logging/__init__.py +++ b/notifications_utils/logging/__init__.py @@ -2,10 +2,10 @@ import logging.handlers import sys import time +from collections.abc import Sequence from itertools import product from os import getpid from pathlib import Path -from typing import Sequence from flask import current_app, g, request from flask.ctx import has_app_context, has_request_context diff --git a/notifications_utils/markdown.py b/notifications_utils/markdown.py index 96bddc4d1..8b2bfcf53 100644 --- a/notifications_utils/markdown.py +++ b/notifications_utils/markdown.py @@ -23,17 +23,13 @@ r"^( *)([•*-]|\d+\.)[\s\S]+?" r"(?:" r"\n+(?=\1?(?:[-*_] *){3,}(?:\n+|$))" # hrule - r"|\n+(?=%s)" # def links - r"|\n+(?=%s)" # def footnotes + rf"|\n+(?={mistune._pure_pattern(mistune.BlockGrammar.def_links)})" # def links + rf"|\n+(?={mistune._pure_pattern(mistune.BlockGrammar.def_footnotes)})" # def footnotes r"|\n{2,}" r"(?! )" r"(?!\1(?:[•*-]|\d+\.) )\n*" r"|" r"\s*$)" - % ( - mistune._pure_pattern(mistune.BlockGrammar.def_links), - mistune._pure_pattern(mistune.BlockGrammar.def_footnotes), - ) ) mistune.BlockGrammar.list_item = re.compile( r"^(( *)(?:[•*-]|\d+\.)[^\n]*" r"(?:\n(?!\2(?:[•*-]|\d+\.))[^\n]*)*)", flags=re.M diff --git a/notifications_utils/recipient_validation/email_address.py b/notifications_utils/recipient_validation/email_address.py index c17055286..7083c0a98 100644 --- a/notifications_utils/recipient_validation/email_address.py +++ b/notifications_utils/recipient_validation/email_address.py @@ -12,7 +12,7 @@ hostname_part = re.compile(r"^(xn|[a-z0-9]+)(-?-[a-z0-9]+)*$", re.IGNORECASE) tld_part = re.compile(r"^([a-z]{2,63}|xn--([a-z0-9]+-)*[a-z0-9]+)$", re.IGNORECASE) VALID_LOCAL_CHARS = r"a-zA-Z0-9.!#$%&'*+/=?^_`{|}~\-" -EMAIL_REGEX_PATTERN = r"^[{}]+@([^.@][^@\s]+)$".format(VALID_LOCAL_CHARS) +EMAIL_REGEX_PATTERN = rf"^[{VALID_LOCAL_CHARS}]+@([^.@][^@\s]+)$" def validate_email_address(email_address): # noqa (C901 too complex) diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index 8ece2a69c..f765e1788 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -4,7 +4,7 @@ from functools import lru_cache from io import StringIO from itertools import islice -from typing import Optional, cast +from typing import cast from ordered_set import OrderedSet @@ -264,11 +264,9 @@ def duplicate_recipient_column_headers(self): ] return OrderedSet( - ( - column_header - for column_header in self._raw_column_headers - if raw_recipient_column_headers.count(InsensitiveDict.make_key(column_header)) > 1 - ) + column_header + for column_header in self._raw_column_headers + if raw_recipient_column_headers.count(InsensitiveDict.make_key(column_header)) > 1 ) def is_address_column(self, key): @@ -369,7 +367,7 @@ def __init__( else: self.message_too_long = template.is_message_too_long() self.message_empty = template.is_message_empty() - self.qr_code_too_long: Optional[QrCodeTooLong] = self._has_qr_code_with_too_much_data() + self.qr_code_too_long: QrCodeTooLong | None = self._has_qr_code_with_too_much_data() super().__init__({key: Cell(key, value, error_fn, self.placeholders) for key, value in row_dict.items()}) @@ -395,7 +393,7 @@ def has_bad_recipient(self) -> bool: def has_bad_postal_address(self): return self.template_type == "letter" and not self.as_postal_address.valid - def _has_qr_code_with_too_much_data(self) -> Optional[QrCodeTooLong]: + def _has_qr_code_with_too_much_data(self) -> QrCodeTooLong | None: if not self._template: return None diff --git a/notifications_utils/request_helper.py b/notifications_utils/request_helper.py index e22fe75bc..07fd328f6 100644 --- a/notifications_utils/request_helper.py +++ b/notifications_utils/request_helper.py @@ -112,7 +112,7 @@ def get_onwards_request_headers(self): ) -class ResponseHeaderMiddleware(object): +class ResponseHeaderMiddleware: def __init__(self, app, trace_id_header, span_id_header): self.app = app self.trace_id_header = trace_id_header diff --git a/notifications_utils/safe_string.py b/notifications_utils/safe_string.py index 7d32041e8..a313f6307 100644 --- a/notifications_utils/safe_string.py +++ b/notifications_utils/safe_string.py @@ -3,7 +3,7 @@ from functools import lru_cache -@lru_cache() +@lru_cache def make_string_safe(string, whitespace): # strips accents, diacritics etc string = "".join(c for c in unicodedata.normalize("NFD", string) if unicodedata.category(c) != "Mn") diff --git a/notifications_utils/template.py b/notifications_utils/template.py index 41197dbc6..755d680a4 100644 --- a/notifications_utils/template.py +++ b/notifications_utils/template.py @@ -4,7 +4,7 @@ from functools import lru_cache from html import unescape from os import path -from typing import Literal, Optional +from typing import Literal from jinja2 import Environment, FileSystemLoader from markupsafe import Markup @@ -656,7 +656,7 @@ def too_many_pages(self): def postal_address(self): return PostalAddress.from_personalisation(InsensitiveDict(self.values)) - def has_qr_code_with_too_much_data(self) -> Optional[QrCodeTooLong]: + def has_qr_code_with_too_much_data(self) -> QrCodeTooLong | None: content = self._personalised_content if self.values else self.content try: Take(content).then(notify_letter_qrcode_validator) diff --git a/notifications_utils/testing/comparisons.py b/notifications_utils/testing/comparisons.py index eb02b3a63..780690867 100644 --- a/notifications_utils/testing/comparisons.py +++ b/notifications_utils/testing/comparisons.py @@ -1,7 +1,7 @@ import re from functools import lru_cache +from re import Pattern from types import MappingProxyType -from typing import Pattern class RestrictedAny: @@ -70,7 +70,7 @@ def __init__(self, *args, **kwargs): self._regex = ( args[0] if len(args) == 1 and isinstance(args[0], Pattern) else self._cached_re_compile(*args, **kwargs) ) - super().__init__(lambda other: isinstance(other, (str, bytes)) and bool(self._regex.match(other))) + super().__init__(lambda other: isinstance(other, str | bytes) and bool(self._regex.match(other))) def __repr__(self): return f"{self.__class__.__name__}({self._regex})" diff --git a/pyproject.toml b/pyproject.toml index afeb9532a..35d4a7a7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ lint.select = [ "C90", # mccabe cyclomatic complexity "G", # flake8-logging-format "T20", # flake8-print + "UP", # pyupgrade ] lint.ignore = [] exclude = [ diff --git a/tests/test_recipient_csv.py b/tests/test_recipient_csv.py index 6fcd6a25d..8bd279230 100644 --- a/tests/test_recipient_csv.py +++ b/tests/test_recipient_csv.py @@ -1166,17 +1166,15 @@ def test_address_validation_speed(): # a second – if it starts to get slow, something is inefficient number_of_lines = 1000 uk_addresses_with_valid_postcodes = "\n".join( - ( - "{n} Example Street, London, {a}{b} {c}{d}{e}".format( - n=randrange(1000), - a=choice(["n", "e", "sw", "se", "w"]), - b=choice(range(1, 10)), - c=choice(range(1, 10)), - d=choice("ABDefgHJLNPqrstUWxyZ"), - e=choice("ABDefgHJLNPqrstUWxyZ"), - ) - for i in range(number_of_lines) + "{n} Example Street, London, {a}{b} {c}{d}{e}".format( + n=randrange(1000), + a=choice(["n", "e", "sw", "se", "w"]), + b=choice(range(1, 10)), + c=choice(range(1, 10)), + d=choice("ABDefgHJLNPqrstUWxyZ"), + e=choice("ABDefgHJLNPqrstUWxyZ"), ) + for i in range(number_of_lines) ) recipients = RecipientCSV( "address line 1, address line 2, address line 3\n" + (uk_addresses_with_valid_postcodes), @@ -1189,14 +1187,8 @@ def test_address_validation_speed(): def test_email_validation_speed(): email_addresses = "\n".join( - ( - "{a}{b}@example-{n}.com,Example,Thursday".format( - n=randrange(1000), - a=choice(string.ascii_letters), - b=choice(string.ascii_letters), - ) - for i in range(1000) - ) + f"{choice(string.ascii_letters)}{choice(string.ascii_letters)}@example-{randrange(1000)}.com,Example,Thursday" + for i in range(1000) ) recipients = RecipientCSV( "email address,name,day\n" + email_addresses, diff --git a/tests/test_serialised_model.py b/tests/test_serialised_model.py index 6c620775e..a07bc06d2 100644 --- a/tests/test_serialised_model.py +++ b/tests/test_serialised_model.py @@ -79,12 +79,9 @@ def foo(self): with pytest.raises(AttributeError) as e: Custom({"foo": "NOPE"}) - if sys.version_info < (3, 11): - assert str(e.value) == "can't set attribute" - else: - assert str(e.value) == ( - "property 'foo' of 'test_cant_override_custom_property_from_dict..Custom' object has no setter" - ) + assert str(e.value) == ( + "property 'foo' of 'test_cant_override_custom_property_from_dict..Custom' object has no setter" + ) @pytest.mark.parametrize( From 38d37f09cb9472e60888bd5117f4e3b89c628b9f Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 9 May 2024 10:11:40 +0100 Subject: [PATCH 052/211] Add flake8-comprehensions linter --- .../clients/zendesk/zendesk_client.py | 2 +- .../international_billing_rates.py | 2 +- notifications_utils/logging/__init__.py | 2 +- notifications_utils/markdown.py | 28 ++++++++----------- notifications_utils/recipients.py | 4 +-- notifications_utils/sanitise_text.py | 2 +- notifications_utils/template.py | 4 +-- pyproject.toml | 1 + tests/test_recipient_csv.py | 10 +++---- tests/test_template_change.py | 8 +++--- 10 files changed, 30 insertions(+), 33 deletions(-) diff --git a/notifications_utils/clients/zendesk/zendesk_client.py b/notifications_utils/clients/zendesk/zendesk_client.py index 43ad531bd..a84f16536 100644 --- a/notifications_utils/clients/zendesk/zendesk_client.py +++ b/notifications_utils/clients/zendesk/zendesk_client.py @@ -41,7 +41,7 @@ class NotifySupportTicketComment: body: str # A list of file-like objects to attach to the comment - attachments: typing.Sequence[NotifySupportTicketAttachment] = tuple() + attachments: typing.Sequence[NotifySupportTicketAttachment] = () # Whether the comment is public or internal public: bool = True diff --git a/notifications_utils/international_billing_rates.py b/notifications_utils/international_billing_rates.py index c9e5dc292..eae5b76a2 100644 --- a/notifications_utils/international_billing_rates.py +++ b/notifications_utils/international_billing_rates.py @@ -23,4 +23,4 @@ import yaml INTERNATIONAL_BILLING_RATES = yaml.safe_load((Path(__file__).parent / "international_billing_rates.yml").read_text()) -COUNTRY_PREFIXES = list(reversed(sorted(INTERNATIONAL_BILLING_RATES.keys(), key=len))) +COUNTRY_PREFIXES = sorted(INTERNATIONAL_BILLING_RATES.keys(), key=len, reverse=True) diff --git a/notifications_utils/logging/__init__.py b/notifications_utils/logging/__init__.py index 813d4bbe9..a251265ee 100644 --- a/notifications_utils/logging/__init__.py +++ b/notifications_utils/logging/__init__.py @@ -40,7 +40,7 @@ def _common_request_extra_log_context(): } -def init_app(app, statsd_client=None, extra_filters: Sequence[logging.Filter] = tuple()): +def init_app(app, statsd_client=None, extra_filters: Sequence[logging.Filter] = ()): app.config.setdefault("NOTIFY_LOG_LEVEL", "INFO") app.config.setdefault("NOTIFY_APP_NAME", "none") app.config.setdefault("NOTIFY_LOG_DEBUG_PATH_LIST", {"/_status", "/metrics"}) diff --git a/notifications_utils/markdown.py b/notifications_utils/markdown.py index 8b2bfcf53..add1b5529 100644 --- a/notifications_utils/markdown.py +++ b/notifications_utils/markdown.py @@ -39,25 +39,21 @@ mistune.InlineLexer.default_rules = list( OrderedSet(mistune.InlineLexer.default_rules) - - set( - ( - "emphasis", - "double_emphasis", - "strikethrough", - "code", - ) - ) + - { + "emphasis", + "double_emphasis", + "strikethrough", + "code", + } ) mistune.InlineLexer.inline_html_rules = list( set(mistune.InlineLexer.inline_html_rules) - - set( - ( - "emphasis", - "double_emphasis", - "strikethrough", - "code", - ) - ) + - { + "emphasis", + "double_emphasis", + "strikethrough", + "code", + } ) diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index f765e1788..9500ff689 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -246,14 +246,14 @@ def column_headers_as_column_keys(self): @property def missing_column_headers(self): - return set( + return { key for key in self.placeholders if ( InsensitiveDict.make_key(key) not in self.column_headers_as_column_keys and not self.is_address_column(key) ) - ) + } @property def duplicate_recipient_column_headers(self): diff --git a/notifications_utils/sanitise_text.py b/notifications_utils/sanitise_text.py index 551b75b36..dcea2d75c 100644 --- a/notifications_utils/sanitise_text.py +++ b/notifications_utils/sanitise_text.py @@ -38,7 +38,7 @@ def get_non_compatible_characters(cls, content): This follows the same rules as `cls.encode`, but returns just the characters that encode would replace with `?` """ - return set(c for c in content if c not in cls.ALLOWED_CHARACTERS and cls.downgrade_character(c) is None) + return {c for c in content if c not in cls.ALLOWED_CHARACTERS and cls.downgrade_character(c) is None} @staticmethod def get_unicode_char_from_codepoint(codepoint): diff --git a/notifications_utils/template.py b/notifications_utils/template.py index 755d680a4..66291d547 100644 --- a/notifications_utils/template.py +++ b/notifications_utils/template.py @@ -116,7 +116,7 @@ def values(self, value): placeholders = InsensitiveDict.from_keys(self.placeholders) self._values = InsensitiveDict(value).as_dict_with_keys( self.placeholders - | set(key for key in value.keys() if InsensitiveDict.make_key(key) not in placeholders.keys()) + | {key for key in value.keys() if InsensitiveDict.make_key(key) not in placeholders.keys()} ) @property @@ -130,7 +130,7 @@ def placeholders(self): @property def missing_data(self): - return list(placeholder for placeholder in self.placeholders if self.values.get(placeholder) is None) + return [placeholder for placeholder in self.placeholders if self.values.get(placeholder) is None] @property def additional_data(self): diff --git a/pyproject.toml b/pyproject.toml index 35d4a7a7f..7b989c1ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ lint.select = [ "G", # flake8-logging-format "T20", # flake8-print "UP", # pyupgrade + "C4", # flake8-comprehensions ] lint.ignore = [] exclude = [ diff --git a/tests/test_recipient_csv.py b/tests/test_recipient_csv.py index 8bd279230..6271090da 100644 --- a/tests/test_recipient_csv.py +++ b/tests/test_recipient_csv.py @@ -35,7 +35,7 @@ def _sample_template(template_type, content="foo"): def _index_rows(rows): - return set(row.index for row in rows) + return {row.index for row in rows} @pytest.mark.parametrize( @@ -496,7 +496,7 @@ def test_get_recipient_respects_order(file_contents, template, expected_recipien @pytest.mark.parametrize( "file_contents,template_type,expected,expected_missing", [ - ("", "sms", [], set(["phone number", "name"])), + ("", "sms", [], {"phone number", "name"}), ( """ phone number,name @@ -530,7 +530,7 @@ def test_get_recipient_respects_order(file_contents, template, expected_recipien """, "email", ["email address", "colour"], - set(["name"]), + {"name"}, ), ( """ @@ -1018,7 +1018,7 @@ def test_multiple_sms_recipient_columns(international_sms): allow_international_sms=international_sms, ) assert recipients.column_headers == ["phone number", "phone_number", "foo"] - assert recipients.column_headers_as_column_keys == dict(phonenumber="", foo="").keys() + assert recipients.column_headers_as_column_keys == {"phonenumber": "", "foo": ""}.keys() assert recipients.rows[0].get("phone number").data == ("07900 900333") assert recipients.rows[0].get("phone_number").data == ("07900 900333") assert recipients.rows[0].get("phone number").error is None @@ -1042,7 +1042,7 @@ def test_multiple_sms_recipient_columns_with_missing_data(column_name): if column_name != "phone number": expected_column_headers.append(column_name) assert recipients.column_headers == expected_column_headers - assert recipients.column_headers_as_column_keys == dict(phonenumber="", names="").keys() + assert recipients.column_headers_as_column_keys == {"phonenumber": "", "names": ""}.keys() # A piece of weirdness uncovered: since rows are created before spaces in column names are normalised, when # there are duplicate recipient columns and there is data for only one of the columns, if the columns have the same # spacing, phone number data will be a list of this one phone number and None, while if the spacing style differs diff --git a/tests/test_template_change.py b/tests/test_template_change.py index c9cfc712c..47d3d6e06 100644 --- a/tests/test_template_change.py +++ b/tests/test_template_change.py @@ -33,8 +33,8 @@ def test_checking_for_difference_between_templates(old_template, new_template, s set(), ), (ConcreteTemplate({"content": "((1)) ((2)) ((3))"}), ConcreteTemplate({"content": "((1))"}), set()), - (ConcreteTemplate({"content": "((1))"}), ConcreteTemplate({"content": "((1)) ((2)) ((3))"}), set(["2", "3"])), - (ConcreteTemplate({"content": "((a))"}), ConcreteTemplate({"content": "((A)) ((B)) ((C))"}), set(["B", "C"])), + (ConcreteTemplate({"content": "((1))"}), ConcreteTemplate({"content": "((1)) ((2)) ((3))"}), {"2", "3"}), + (ConcreteTemplate({"content": "((a))"}), ConcreteTemplate({"content": "((A)) ((B)) ((C))"}), {"B", "C"}), ], ) def test_placeholders_added(old_template, new_template, placeholders_added): @@ -51,8 +51,8 @@ def test_placeholders_added(old_template, new_template, placeholders_added): set(), ), (ConcreteTemplate({"content": "((1))"}), ConcreteTemplate({"content": "((1)) ((2)) ((3))"}), set()), - (ConcreteTemplate({"content": "((1)) ((2)) ((3))"}), ConcreteTemplate({"content": "((1))"}), set(["2", "3"])), - (ConcreteTemplate({"content": "((a)) ((b)) ((c))"}), ConcreteTemplate({"content": "((A))"}), set(["b", "c"])), + (ConcreteTemplate({"content": "((1)) ((2)) ((3))"}), ConcreteTemplate({"content": "((1))"}), {"2", "3"}), + (ConcreteTemplate({"content": "((a)) ((b)) ((c))"}), ConcreteTemplate({"content": "((A))"}), {"b", "c"}), ], ) def test_placeholders_removed(old_template, new_template, placeholders_removed): From a8dc6c00113091ed735fa832881ff696fb1f632f Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 9 May 2024 10:14:42 +0100 Subject: [PATCH 053/211] Add flake8-implicit-str-concat linter --- notifications_utils/formatters.py | 6 +-- notifications_utils/logging/formatting.py | 4 +- notifications_utils/markdown.py | 2 +- notifications_utils/recipients.py | 2 +- notifications_utils/sanitise_text.py | 2 +- notifications_utils/template.py | 2 +- pyproject.toml | 1 + .../test_postal_address.py | 12 ++--- tests/test_field.py | 2 +- tests/test_markdown.py | 14 ++--- tests/test_serialised_model.py | 2 +- tests/test_template_types.py | 52 +++++++++---------- 12 files changed, 49 insertions(+), 52 deletions(-) diff --git a/notifications_utils/formatters.py b/notifications_utils/formatters.py index a4c37b548..037e2260d 100644 --- a/notifications_utils/formatters.py +++ b/notifications_utils/formatters.py @@ -21,7 +21,7 @@ "\u2029" # paragraph separator ) -OBSCURE_FULL_WIDTH_WHITESPACE = "\u00A0" "\u202F" # non breaking space # narrow no break space +OBSCURE_FULL_WIDTH_WHITESPACE = "\u00A0\u202F" # non breaking space # narrow no break space ALL_WHITESPACE = string.whitespace + OBSCURE_ZERO_WIDTH_WHITESPACE + OBSCURE_FULL_WIDTH_WHITESPACE @@ -150,7 +150,7 @@ def sms_encode(content): """ Re-implements html._charref but makes trailing semicolons non-optional """ -_charref = re.compile(r"&(#[0-9]+;" r"|#[xX][0-9a-fA-F]+;" r"|[^\t\n\f <&#;]{1,32};)") +_charref = re.compile(r"&(#[0-9]+;|#[xX][0-9a-fA-F]+;|[^\t\n\f <&#;]{1,32};)") def unescape_strict(s): @@ -221,7 +221,7 @@ def make_quotes_smart(value): def replace_hyphens_with_en_dashes(value): return re.sub( hyphens_surrounded_by_spaces, - (" " "\u2013" " "), # space # en dash # space + (" \u2013 "), # space # en dash # space value, ) diff --git a/notifications_utils/logging/formatting.py b/notifications_utils/logging/formatting.py index 92d6ecf29..0e500084c 100644 --- a/notifications_utils/logging/formatting.py +++ b/notifications_utils/logging/formatting.py @@ -3,9 +3,7 @@ from pythonjsonlogger.jsonlogger import JsonFormatter as BaseJSONFormatter -LOG_FORMAT = ( - "%(asctime)s %(app_name)s %(name)s %(levelname)s " '%(request_id)s "%(message)s" [in %(pathname)s:%(lineno)d]' -) +LOG_FORMAT = '%(asctime)s %(app_name)s %(name)s %(levelname)s %(request_id)s "%(message)s" [in %(pathname)s:%(lineno)d]' TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" diff --git a/notifications_utils/markdown.py b/notifications_utils/markdown.py index add1b5529..e67600e39 100644 --- a/notifications_utils/markdown.py +++ b/notifications_utils/markdown.py @@ -32,7 +32,7 @@ r"\s*$)" ) mistune.BlockGrammar.list_item = re.compile( - r"^(( *)(?:[•*-]|\d+\.)[^\n]*" r"(?:\n(?!\2(?:[•*-]|\d+\.))[^\n]*)*)", flags=re.M + r"^(( *)(?:[•*-]|\d+\.)[^\n]*(?:\n(?!\2(?:[•*-]|\d+\.))[^\n]*)*)", flags=re.M ) mistune.BlockGrammar.list_bullet = re.compile(r"^ *(?:[•*-]|\d+\.)") mistune.InlineGrammar.url = re.compile(r"""^(https?:\/\/[^\s<]+[^<.,:"')\]\s])""") diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index 9500ff689..9f71c9d28 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -85,7 +85,7 @@ def template(self): @template.setter def template(self, value): if not isinstance(value, Template): - raise TypeError("template must be an instance of " "notifications_utils.template.Template") + raise TypeError("template must be an instance of notifications_utils.template.Template") self._template = value self.template_type = self._template.template_type self.recipient_column_headers = first_column_headings[self.template_type] diff --git a/notifications_utils/sanitise_text.py b/notifications_utils/sanitise_text.py index dcea2d75c..987021852 100644 --- a/notifications_utils/sanitise_text.py +++ b/notifications_utils/sanitise_text.py @@ -123,7 +123,7 @@ class SanitiseSMS(SanitiseText): GSM_CHARACTERS = ( set( "@£$¥èéùìòÇ\nØø\rÅåΔ_ΦΓΛΩΠΨΣΘΞ\x1bÆæßÉ !\"#¤%&'()*+,-./0123456789:;<=>?" - + "¡ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÑܧ¿abcdefghijklmnopqrstuvwxyzäöñüà" + "¡ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÑܧ¿abcdefghijklmnopqrstuvwxyzäöñüà" ) | EXTENDED_GSM_CHARACTERS ) diff --git a/notifications_utils/template.py b/notifications_utils/template.py index 66291d547..01e60b2ea 100644 --- a/notifications_utils/template.py +++ b/notifications_utils/template.py @@ -73,7 +73,7 @@ def __init__( raise TypeError("Values must be a dict") if template.get("template_type") != self.template_type: raise TypeError( - f"Cannot initialise {self.__class__.__name__} " f'with {template.get("template_type")} template_type' + f'Cannot initialise {self.__class__.__name__} with {template.get("template_type")} template_type' ) self.id = template.get("id", None) self.name = template.get("name", None) diff --git a/pyproject.toml b/pyproject.toml index 7b989c1ac..9798da2b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ lint.select = [ "T20", # flake8-print "UP", # pyupgrade "C4", # flake8-comprehensions + "ISC", # flake8-implicit-str-concat ] lint.ignore = [] exclude = [ diff --git a/tests/recipient_validation/test_postal_address.py b/tests/recipient_validation/test_postal_address.py index d6a0fd258..e81145012 100644 --- a/tests/recipient_validation/test_postal_address.py +++ b/tests/recipient_validation/test_postal_address.py @@ -334,12 +334,12 @@ def test_international(address, expected_international): S W1 A 1 AA """, - ("123 Example St.\n" "City of Town\n" "SW1A 1AA"), + ("123 Example St.\nCity of Town\nSW1A 1AA"), ("123 Example St., City of Town, SW1A 1AA"), ), ( - ("123 Example St. \t , \n" ", , , , , , ,\n" "City of Town, Region,\n" "SW1A 1AA,,\n"), - ("123 Example St.\n" "City of Town, Region\n" "SW1A 1AA"), + ("123 Example St. \t , \n, , , , , , ,\nCity of Town, Region,\nSW1A 1AA,,\n"), + ("123 Example St.\nCity of Town, Region\nSW1A 1AA"), ("123 Example St., City of Town, Region, SW1A 1AA"), ), ( @@ -349,7 +349,7 @@ def test_international(address, expected_international): """, - ("123 Example Straße\n" "Germany"), + ("123 Example Straße\nGermany"), ("123 Example Straße, Germany"), ), ), @@ -439,7 +439,7 @@ def test_postage(address, expected_postage): ) def test_from_personalisation(personalisation): assert PostalAddress.from_personalisation(personalisation).normalised == ( - "123 Example Street\n" "City of Town\n" "SW1A 1AA" + "123 Example Street\nCity of Town\nSW1A 1AA" ) @@ -451,7 +451,7 @@ def test_from_personalisation_handles_int(): "address_line_4": "SW1A1AA", } assert PostalAddress.from_personalisation(personalisation).normalised == ( - "123\n" "Example Street\n" "City of Town\n" "SW1A 1AA" + "123\nExample Street\nCity of Town\nSW1A 1AA" ) diff --git a/tests/test_field.py b/tests/test_field.py index b7542d225..8a03f24ad 100644 --- a/tests/test_field.py +++ b/tests/test_field.py @@ -216,7 +216,7 @@ def test_handling_of_missing_values(content, values, expected): 99.99999, "off", "exclude", - "no" "any random string", + "noany random string", "false", False, [], diff --git a/tests/test_markdown.py b/tests/test_markdown.py index 084d6fa8b..48b1f9730 100644 --- a/tests/test_markdown.py +++ b/tests/test_markdown.py @@ -252,7 +252,7 @@ def test_level_3_header(markdown_function, expected): @pytest.mark.parametrize( "markdown_function, expected", ( - [notify_letter_preview_markdown, ("

a

" '
 
' "

b

")], + [notify_letter_preview_markdown, ('

a

 

b

')], [ notify_email_markdown, ( @@ -474,8 +474,8 @@ def test_autolink(markdown_function, link, expected): notify_email_markdown, ( '

' - + "variable called `thing`" - + "

" + "variable called `thing`" + "

" ), ], [ @@ -496,8 +496,8 @@ def test_codespan(markdown_function, expected): notify_email_markdown, ( '

' - + "something **important**" - + "

" + "something **important**" + "

" ), ], [ @@ -519,8 +519,8 @@ def test_double_emphasis(markdown_function, expected): "something *important*", ( '

' - + "something *important*" - + "

" + "something *important*" + "

" ), ], [ diff --git a/tests/test_serialised_model.py b/tests/test_serialised_model.py index a07bc06d2..18bbd91f7 100644 --- a/tests/test_serialised_model.py +++ b/tests/test_serialised_model.py @@ -55,7 +55,7 @@ class CustomCollection(SerialisedModelCollection): if sys.version_info >= (3, 12): assert str(e.value) == ( - "Can't instantiate abstract class CustomCollection without an implementation " "for abstract method 'model'" + "Can't instantiate abstract class CustomCollection without an implementation for abstract method 'model'" ) else: assert str(e.value) == "Can't instantiate abstract class CustomCollection with abstract method model" diff --git a/tests/test_template_types.py b/tests/test_template_types.py index 6c7c32f87..4a796d58a 100644 --- a/tests/test_template_types.py +++ b/tests/test_template_types.py @@ -318,14 +318,14 @@ def test_content_of_preheader_in_html_emails( HTMLEmailTemplate, "email", {}, - ("the quick brown fox\n" "\n" "jumped over the lazy dog\n"), + ("the quick brown fox\n\njumped over the lazy dog\n"), "notifications_utils.template.notify_email_markdown", ], [ LetterPreviewTemplate, "letter", {}, - ("the quick brown fox\n" "\n" "jumped over the lazy dog\n"), + ("the quick brown fox\n\njumped over the lazy dog\n"), "notifications_utils.template.notify_letter_preview_markdown", ], ], @@ -341,7 +341,7 @@ def test_markdown_in_templates( str( template_class( { - "content": ("the quick ((colour)) ((animal))\n" "\n" "jumped over the lazy dog"), + "content": ("the quick ((colour)) ((animal))\n\njumped over the lazy dog"), "subject": "animal story", "template_type": template_type, }, @@ -628,15 +628,15 @@ def test_sms_preview_adds_newlines(nl2br): @pytest.mark.parametrize( "content", [ - ("one newline\n" "two newlines\n" "\n" "end"), # Unix-style - ("one newline\r\n" "two newlines\r\n" "\r\n" "end"), # Windows-style - ("one newline\r" "two newlines\r" "\r" "end"), # Mac Classic style - ("\t\t\n\r one newline\n" "two newlines\r" "\r\n" "end\n\n \r \n \t "), # A mess + ("one newline\ntwo newlines\n\nend"), # Unix-style + ("one newline\r\ntwo newlines\r\n\r\nend"), # Windows-style + ("one newline\rtwo newlines\r\rend"), # Mac Classic style + ("\t\t\n\r one newline\ntwo newlines\r\r\nend\n\n \r \n \t "), # A mess ], ) def test_sms_message_normalises_newlines(content): assert repr(str(SMSMessageTemplate({"content": content, "template_type": "sms"}))) == repr( - "one newline\n" "two newlines\n" "\n" "end" + "one newline\ntwo newlines\n\nend" ) @@ -1776,7 +1776,7 @@ def test_email_preview_shows_recipient_address( "addressline6": None, "postcode": "N1 4Wq", }, - ("
    " "
  • line 1
    • " "
  • line 2
    • " "
  • N1 4WQ
    • " "
"), + ("
  • line 1
  • line 2
  • N1 4WQ
"), ), ( { @@ -1785,7 +1785,7 @@ def test_email_preview_shows_recipient_address( "addressline3": "\t ,", "postcode": "N1 4WQ", }, - ("
    " "
  • line 1
    • " "
  • line 2
    • " "
  • N1 4WQ
    • " "
"), + ("
  • line 1
  • line 2
  • N1 4WQ
"), ), ( { @@ -1794,7 +1794,7 @@ def test_email_preview_shows_recipient_address( "postcode": "SW1A 1AA", # ignored in favour of line 7 "addressline7": "N1 4WQ", }, - ("
    " "
  • line 1
    • " "
  • line 2
    • " "
  • N1 4WQ
    • " "
"), + ("
  • line 1
  • line 2
  • N1 4WQ
"), ), ( { @@ -1802,7 +1802,7 @@ def test_email_preview_shows_recipient_address( "addressline2": "line 2", "addressline7": "N1 4WQ", # means postcode isn’t needed }, - ("
    " "
  • line 1
    • " "
  • line 2
    • " "
  • N1 4WQ
    • " "
"), + ("
  • line 1
  • line 2
  • N1 4WQ
"), ), ], ) @@ -1825,16 +1825,16 @@ def test_letter_address_format(template_class, address, expected): "markdown, expected", [ ( - ("Here is a list of bullets:\n" "\n" "* one\n" "* two\n" "* three\n" "\n" "New paragraph"), - ("
    \n" "
  • one
    • \n" "
  • two
    • \n" "
  • three
    • \n" "
\n" "

New paragraph

\n"), + ("Here is a list of bullets:\n\n* one\n* two\n* three\n\nNew paragraph"), + ("
    \n
  • one
    • \n
  • two
    • \n
  • three
    • \n
\n

New paragraph

\n"), ), ( - ("# List title:\n" "\n" "* one\n" "* two\n" "* three\n"), - ("

List title:

\n" "
    \n" "
  • one
    • \n" "
  • two
    • \n" "
  • three
    • \n" "
\n"), + ("# List title:\n\n* one\n* two\n* three\n"), + ("

List title:

\n
    \n
  • one
    • \n
  • two
    • \n
  • three
    • \n
\n"), ), ( - ("Here’s an ordered list:\n" "\n" "1. one\n" "2. two\n" "3. three\n"), - ("

Here’s an ordered list:

    \n" "
  1. one
    2. \n" "
  2. two
    3. \n" "
  3. three
    4. \n" "
"), + ("Here’s an ordered list:\n\n1. one\n2. two\n3. three\n"), + ("

Here’s an ordered list:

    \n
  1. one
    2. \n
  2. two
    3. \n
  3. three
    4. \n
"), ), ], ) @@ -1941,14 +1941,14 @@ def test_message_too_long_for_an_email_message_within_limits(template_class, tem @pytest.mark.parametrize( - ("content," "expected_preview_markup,"), + ("content,expected_preview_markup,"), [ ( "a\n\n\nb", - ("

a

" "

b

"), + ("

a

b

"), ), ( - ("a\n" "\n" "* one\n" "* two\n" "* three\n" "and a half\n" "\n" "\n" "\n" "\n" "foo"), + ("a\n\n* one\n* two\n* three\nand a half\n\n\n\n\nfoo"), ( "

a

    \n" "
  • one
    • \n" @@ -2253,7 +2253,7 @@ def test_image_not_present_if_no_logo(template_class): @pytest.mark.parametrize( "content", ( - ("The quick brown fox.\n" "\n\n\n\n" "Jumps over the lazy dog. \n" "Single linebreak above."), + ("The quick brown fox.\n\n\n\n\nJumps over the lazy dog. \nSingle linebreak above."), ( "\n \n" "The quick brown fox. \n\n" @@ -2267,9 +2267,9 @@ def test_image_not_present_if_no_logo(template_class): ( ( SMSBodyPreviewTemplate, - ("The quick brown fox.\n" "\n" "Jumps over the lazy dog.\n" "Single linebreak above."), + ("The quick brown fox.\n\nJumps over the lazy dog.\nSingle linebreak above."), ), - (SMSMessageTemplate, ("The quick brown fox.\n" "\n" "Jumps over the lazy dog.\n" "Single linebreak above.")), + (SMSMessageTemplate, ("The quick brown fox.\n\nJumps over the lazy dog.\nSingle linebreak above.")), ( SMSPreviewTemplate, ( @@ -2289,9 +2289,7 @@ def test_text_messages_collapse_consecutive_whitespace( template = template_class({"content": content, "template_type": "sms"}) assert str(template) == expected assert ( - template.content_count - == 70 - == len("The quick brown fox.\n" "\n" "Jumps over the lazy dog.\n" "Single linebreak above.") + template.content_count == 70 == len("The quick brown fox.\n\nJumps over the lazy dog.\nSingle linebreak above.") ) From 8016ce3a6b16f3d30ed2ff6731fd20851c2b4803 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 9 May 2024 10:15:09 +0100 Subject: [PATCH 054/211] Add flake8-raise linter --- pyproject.toml | 1 + tests/test_request_id.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9798da2b7..747ec9edf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ lint.select = [ "UP", # pyupgrade "C4", # flake8-comprehensions "ISC", # flake8-implicit-str-concat + "RSE", # flake8-raise ] lint.ignore = [] exclude = [ diff --git a/tests/test_request_id.py b/tests/test_request_id.py index 95ab53737..88bc18d75 100644 --- a/tests/test_request_id.py +++ b/tests/test_request_id.py @@ -26,7 +26,7 @@ def test_request_id_is_set_on_error_response(app): @app.route("/") def error_route(): - raise Exception() + raise Exception with app.app_context(): response = client.get("/", headers={"X-B3-TraceId": "generated", "X-B3-SpanId": "generated"}) @@ -483,7 +483,7 @@ def test_response_headers_error_response( @app.route("/") def error_route(): - raise Exception() + raise Exception with app.app_context(): response = client.get("/", headers=extra_req_headers) From 71483b5fb0251dd9e99f6cd45060df3c934cd7c1 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 9 May 2024 10:15:54 +0100 Subject: [PATCH 055/211] Add flake8-pie linter --- notifications_utils/clients/statsd/statsd_client.py | 1 - pyproject.toml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/notifications_utils/clients/statsd/statsd_client.py b/notifications_utils/clients/statsd/statsd_client.py index 6f386d229..f7781f783 100644 --- a/notifications_utils/clients/statsd/statsd_client.py +++ b/notifications_utils/clients/statsd/statsd_client.py @@ -41,7 +41,6 @@ def _send(self, data): self._sock.sendto(data.encode("ascii"), (host, self._port)) except Exception as e: current_app.logger.warning("Error sending statsd metric: %s", e) - pass class StatsdClient: diff --git a/pyproject.toml b/pyproject.toml index 747ec9edf..02dd52f90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ lint.select = [ "C4", # flake8-comprehensions "ISC", # flake8-implicit-str-concat "RSE", # flake8-raise + "PIE", # flake8-pie ] lint.ignore = [] exclude = [ From 838f066f82bfe7c66092d56c75ed09e539d925b4 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 3 Jun 2024 14:47:20 +0100 Subject: [PATCH 056/211] Major version bump to 79.0.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de399bddc..2724b24d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 79.0.0 + +* Switches on Pyupgrade and a bunch of other more opinionated linting rules + ## 78.2.0 * Bumped minimum versions of select subdependencies diff --git a/notifications_utils/version.py b/notifications_utils/version.py index c45f721c8..3b998fc66 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "78.2.0" # 47dcea615e142c6d524dde2064804fbd +__version__ = "79.0.0" # a19819b5ebe4704a9890ed189616f09f From fd36495a2495249ccc2386441c124f0855eae21b Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 19 Jun 2024 10:02:03 +0100 Subject: [PATCH 057/211] Update tests/test_field.py Co-authored-by: Leo Hemsted --- tests/test_field.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_field.py b/tests/test_field.py index 8a03f24ad..5f58ea51e 100644 --- a/tests/test_field.py +++ b/tests/test_field.py @@ -216,7 +216,8 @@ def test_handling_of_missing_values(content, values, expected): 99.99999, "off", "exclude", - "noany random string", + "no", + "any random string", "false", False, [], From ed82ae989439647d06ccf0d099b5597addd9987a Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 19 Jun 2024 10:03:31 +0100 Subject: [PATCH 058/211] Remove unused exception We dont appear to use this anywhere (and if we need to catch elsewhere, those libs can just import redis.exceptions) --- notifications_utils/clients/redis/redis_client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/notifications_utils/clients/redis/redis_client.py b/notifications_utils/clients/redis/redis_client.py index 24736737f..009f0435b 100644 --- a/notifications_utils/clients/redis/redis_client.py +++ b/notifications_utils/clients/redis/redis_client.py @@ -9,9 +9,6 @@ from flask import current_app from flask_redis import FlaskRedis - -# expose redis exceptions so that they can be caught -from redis.exceptions import RedisError # noqa: F401 from redis.lock import Lock from redis.typing import Number From 27311ac7d952fb067dc93a0eb279462ebb4c9ef4 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 19 Jun 2024 10:06:51 +0100 Subject: [PATCH 059/211] Ignore linter-generated changes in git blame --- .git-blame-ignore-revs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 62d41727e..f4fe74780 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,10 @@ # bulk change for black formatting changes 0f3f4b3e6aa5a199a1b5e79742aa776ce8d8bf7e ce619c18021e1a219cac2da4e3b0470ddfc08508 + +# adding more linters to ruff +26c86b6a8814b0f5948d9f1ea271d67141dbf50e +38d37f09cb9472e60888bd5117f4e3b89c628b9f +a8dc6c00113091ed735fa832881ff696fb1f632f +8016ce3a6b16f3d30ed2ff6731fd20851c2b4803 +71483b5fb0251dd9e99f6cd45060df3c934cd7c1 From 5944fb075b305bb10d186d2459f457d982315121 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Fri, 28 Jun 2024 16:52:31 +0100 Subject: [PATCH 060/211] Return Zendesk ticket ID when creating a ticket In order to update a ticket, you need to know its ID. This returns the ticket ID from the `send_ticket_to_zendesk` method so that we can update a ticket immediately after creating it. We want to start adding an internal note if a user who is not logged in creates a ticket. --- notifications_utils/clients/zendesk/zendesk_client.py | 2 ++ tests/clients/zendesk/test_zendesk_client.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/notifications_utils/clients/zendesk/zendesk_client.py b/notifications_utils/clients/zendesk/zendesk_client.py index a84f16536..c3d3f0a5a 100644 --- a/notifications_utils/clients/zendesk/zendesk_client.py +++ b/notifications_utils/clients/zendesk/zendesk_client.py @@ -80,6 +80,8 @@ def send_ticket_to_zendesk(self, ticket): current_app.logger.info("Zendesk create ticket %s succeeded", ticket_id) + return ticket_id + def _is_user_suspended(self, response): requester_error = response["details"].get("requester") return requester_error and ("suspended" in requester_error[0]["description"]) diff --git a/tests/clients/zendesk/test_zendesk_client.py b/tests/clients/zendesk/test_zendesk_client.py index 0c40f25cc..1a674fe0c 100644 --- a/tests/clients/zendesk/test_zendesk_client.py +++ b/tests/clients/zendesk/test_zendesk_client.py @@ -42,13 +42,14 @@ def test_zendesk_client_send_ticket_to_zendesk(zendesk_client, app, rmock, caplo with caplog.at_level(logging.INFO): ticket = NotifySupportTicket("subject", "message", "incident") - zendesk_client.send_ticket_to_zendesk(ticket) + response = zendesk_client.send_ticket_to_zendesk(ticket) assert rmock.last_request.headers["Authorization"][:6] == "Basic " b64_auth = rmock.last_request.headers["Authorization"][6:] assert b64decode(b64_auth.encode()).decode() == "zd-api-notify@digital.cabinet-office.gov.uk/token:testkey" assert rmock.last_request.json() == ticket.request_data assert "Zendesk create ticket 12345 succeeded" in caplog.messages + assert response == 12345 def test_zendesk_client_send_ticket_to_zendesk_error(zendesk_client, app, rmock, caplog): @@ -74,12 +75,13 @@ def test_zendesk_client_send_ticket_to_zendesk_with_user_suspended_error(zendesk }, ) ticket = NotifySupportTicket("subject", "message", "incident") - zendesk_client.send_ticket_to_zendesk(ticket) + response = zendesk_client.send_ticket_to_zendesk(ticket) assert caplog.messages == [ "Zendesk create ticket failed because user is suspended " "'{'requester': [{'description': 'Requester: Joe Bloggs is suspended.'}]}'" ] + assert response is None @pytest.mark.parametrize( From 458215b595bc819aa10597b5217f83ad8fad1bc8 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Fri, 28 Jun 2024 16:56:13 +0100 Subject: [PATCH 061/211] Patch version bump to 79.0.1 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2724b24d7..48aa2f203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 79.0.1 + +* Update the `send_ticket_to_zendesk` method of the ZendeskClient to return the ID of the ticket that was created. + ## 79.0.0 * Switches on Pyupgrade and a bunch of other more opinionated linting rules diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 3b998fc66..a0a84388d 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "79.0.0" # a19819b5ebe4704a9890ed189616f09f +__version__ = "79.0.1" # c3bc84a6e9be276955ecaad95d0a0caf From c9aa50047cc269eb51109fb5d9853f01abfba0bd Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 26 Jun 2024 10:59:07 +0100 Subject: [PATCH 062/211] Add common dependencies to `requirements.in` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most of our apps require Gunicorn and Boto. The apps that require Gunicorn require it with the eventlet extension. One app (Document Download API) that requires Boto requires it built with the CRT extension (Amazon’s common runtime environment) so that it can talk to S3 buckets in other regions. But I don’t think it hurts other apps to also use this extension. By specifying them here we can avoid repeatedly specifying them in each app’s `requirements.in` files. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 29db929fb..acdb59572 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "requests>=2.32.0", "python-json-logger>=2.0.7", "Flask>=3.0.0", - "gunicorn>=20.0.0", + "gunicorn[eventlet]>=21.2.0", "ordered-set>=4.1.0", "Jinja2>=3.1.4", "statsd>=4.0.1", @@ -40,7 +40,7 @@ "pypdf>=3.13.0", "itsdangerous>=2.1.2", "govuk-bank-holidays>=0.14", - "boto3>=1.34.100", + "boto3[crt]>=1.34.100", "segno>=1.5.2,<2.0.0", ], ) From 9cfe813550a55a929a18610ffb2c3ef400e56146 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 26 Jun 2024 11:15:47 +0100 Subject: [PATCH 063/211] Rationalise test dependencies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `flake8-` dependencies aren’t used any more because they are built into ruff. The `pytest-profiling` and `snakeviz` dependencies are not used automatically - developers who want to use them can install them manually. This commit also groups Redis and Celery together as similar things. --- requirements_for_test.txt | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/requirements_for_test.txt b/requirements_for_test.txt index e989380c9..879b5e076 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,19 +1,17 @@ -e . -flask>=3.0.0 # we use 2.3.2 in some places but we need >=3 for the logger tests celery==5.3.6 +redis>=4.3.4 # Earlier versions of redis miss features the tests need + beautifulsoup4==4.11.1 pytest==7.2.0 +pytest-env==0.8.1 pytest-mock==3.9.0 pytest-xdist==3.0.2 -requests-mock==1.10.0 -freezegun==1.2.2 -flake8-bugbear==22.10.27 -flake8-print==5.0.0 -pytest-profiling==1.7.0 pytest-testmon==2.1.0 pytest-watch==4.2.0 -redis>=4.3.4 # Earlier versions of redis miss features the tests need -snakeviz==2.1.1 +requests-mock==1.10.0 +freezegun==1.2.2 + black==24.4.0 # Also update `.pre-commit-config.yaml` if this changes ruff==0.3.7 # Also update `.pre-commit-config.yaml` if this changes From 69e5059c1d42aef83c09dffa5a71c23e0bf0ed93 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 26 Jun 2024 11:16:49 +0100 Subject: [PATCH 064/211] Split common test requirements into their own file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So that apps can also use this file, let’s separate it from the test dependencies that only the utils repo uses. --- requirements_for_test.txt | 14 +------------- requirements_for_test_common.txt | 12 ++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 requirements_for_test_common.txt diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 879b5e076..58632853b 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,17 +1,5 @@ -e . +-r requirements_for_test_common.txt celery==5.3.6 redis>=4.3.4 # Earlier versions of redis miss features the tests need - -beautifulsoup4==4.11.1 -pytest==7.2.0 -pytest-env==0.8.1 -pytest-mock==3.9.0 -pytest-xdist==3.0.2 -pytest-testmon==2.1.0 -pytest-watch==4.2.0 -requests-mock==1.10.0 -freezegun==1.2.2 - -black==24.4.0 # Also update `.pre-commit-config.yaml` if this changes -ruff==0.3.7 # Also update `.pre-commit-config.yaml` if this changes diff --git a/requirements_for_test_common.txt b/requirements_for_test_common.txt new file mode 100644 index 000000000..4e69f1f33 --- /dev/null +++ b/requirements_for_test_common.txt @@ -0,0 +1,12 @@ +beautifulsoup4==4.11.1 +pytest==7.2.0 +pytest-env==0.8.1 +pytest-mock==3.9.0 +pytest-xdist==3.0.2 +pytest-testmon==2.1.0 +pytest-watch==4.2.0 +requests-mock==1.10.0 +freezegun==1.2.2 + +black==24.4.0 # Also update `.pre-commit-config.yaml` if this changes +ruff==0.3.7 # Also update `.pre-commit-config.yaml` if this changes From eff4f367f4b4b8851ab2dffbad4d119ad91a0268 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 26 Jun 2024 11:17:35 +0100 Subject: [PATCH 065/211] Rename command to be more generic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It’s going to be about more than just one file in the future --- notifications_utils/version_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notifications_utils/version_tools.py b/notifications_utils/version_tools.py index 29fb485dd..69cba8efd 100644 --- a/notifications_utils/version_tools.py +++ b/notifications_utils/version_tools.py @@ -80,7 +80,7 @@ def get_file_contents_from_github(branch_or_tag, path): return requests.get(f"https://raw.githubusercontent.com/{repo_name}/{branch_or_tag}/{path}").text -def copy_pyproject_toml(): +def copy_config(): local_utils_version = get_app_version() remote_contents = get_file_contents_from_github(local_utils_version, "pyproject.toml") pyproject_file.write_text( From d2676e0d19bc0b67666e9f643d6be2bcdf16cd86 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 26 Jun 2024 11:19:20 +0100 Subject: [PATCH 066/211] Copy common test requirements into apps So that apps can automatically share the same dependencies, and when we want to upgrade a new dependency we only have to do it in one place. --- notifications_utils/version_tools.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/notifications_utils/version_tools.py b/notifications_utils/version_tools.py index 69cba8efd..780a0cb21 100644 --- a/notifications_utils/version_tools.py +++ b/notifications_utils/version_tools.py @@ -4,8 +4,11 @@ requirements_file = pathlib.Path("requirements.in") frozen_requirements_file = pathlib.Path("requirements.txt") -pyproject_file = pathlib.Path("pyproject.toml") repo_name = "alphagov/notifications-utils" +config_files = { + "pyproject.toml", + "requirements_for_test_common.txt", +} class color: @@ -82,7 +85,8 @@ def get_file_contents_from_github(branch_or_tag, path): def copy_config(): local_utils_version = get_app_version() - remote_contents = get_file_contents_from_github(local_utils_version, "pyproject.toml") - pyproject_file.write_text( - f"# This file is automatically copied from notifications-utils@{local_utils_version}\n\n{remote_contents}" - ) + for config_file in config_files: + remote_contents = get_file_contents_from_github(local_utils_version, config_file) + pathlib.Path(config_file).write_text( + f"# This file is automatically copied from notifications-utils@{local_utils_version}\n\n{remote_contents}" + ) From e84b57b14fac0b57ea39621f8f76d13b70836beb Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 26 Jun 2024 11:21:53 +0100 Subject: [PATCH 067/211] Add `.pre-commit-config.yaml` to common config files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So we don’t have to manually update it in the apps when we bump the version of `ruff` or `black`. --- notifications_utils/version_tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/notifications_utils/version_tools.py b/notifications_utils/version_tools.py index 780a0cb21..501f249d6 100644 --- a/notifications_utils/version_tools.py +++ b/notifications_utils/version_tools.py @@ -8,6 +8,7 @@ config_files = { "pyproject.toml", "requirements_for_test_common.txt", + ".pre-commit-config.yaml", } From 6ce6fbcc66970c44a5604bce9bf1799ad4d93d3a Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 26 Jun 2024 11:23:18 +0100 Subject: [PATCH 068/211] Major version bump --- CHANGELOG.md | 5 +++++ notifications_utils/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48aa2f203..6d7dd6465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG +## 80.0.0 + +* Copies additional config files from utils into repos +* Renames `version_tools.copy_pyproject_yaml` to `version_tools.copy_config` + ## 79.0.1 * Update the `send_ticket_to_zendesk` method of the ZendeskClient to return the ID of the ticket that was created. diff --git a/notifications_utils/version.py b/notifications_utils/version.py index a0a84388d..4c844b6ee 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "79.0.1" # c3bc84a6e9be276955ecaad95d0a0caf +__version__ = "80.0.0" # 9fc05f28af962334e7c9b1f089bb534b From 27c7c0f61d490511cc0d4aa67557fdc3f7d6269e Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 1 Jul 2024 17:13:56 +0100 Subject: [PATCH 069/211] Reduce minimum required version of Gunicorn The API has the Gunicorn version pinned here: https://github.com/alphagov/notifications-api/blob/e175f90d8f56c4a1190f2c29dfa1954ff6b87e64/requirements.in#L13 This resolves to version 20.1.0 here: https://github.com/benoitc/gunicorn/blob/1299ea9e967a61ae2edebe191082fd169b864c64/gunicorn/__init__.py#L6C16-L6C26 So to avoid a dependency conflict we need to reduce the required version to a common baseline in this repo. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index acdb59572..56df70091 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ "requests>=2.32.0", "python-json-logger>=2.0.7", "Flask>=3.0.0", - "gunicorn[eventlet]>=21.2.0", + "gunicorn[eventlet]>=20.1.0", "ordered-set>=4.1.0", "Jinja2>=3.1.4", "statsd>=4.0.1", From 7d1a6ec19570a86fa1057d8313f719ad720e8df8 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Tue, 2 Jul 2024 11:35:20 +0100 Subject: [PATCH 070/211] Patch version bump --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d7dd6465..75e5265f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 80.0.1 + +* Reduces minimum required Gunicorn version for compatibility + ## 80.0.0 * Copies additional config files from utils into repos diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 4c844b6ee..54fb685b4 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "80.0.0" # 9fc05f28af962334e7c9b1f089bb534b +__version__ = "80.0.1" # 5d80aecac00f9145ca76d05c62848a87 From 67492ddd0ce1fb73a62544d80d8acef1aca1eee9 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 12 Jun 2024 00:28:08 +0100 Subject: [PATCH 071/211] add more UK-based test cases ahead split out landlines into valid and invalid, and include some esoteric valid and invalid UK numbers courtesy of wikipedia. Note these are all invalid at the moment as we don't yet support non-mobile UK phone numbers --- .../recipient_validation/test_phone_number.py | 58 ++++++++++++------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index d067a5224..74144140b 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -16,7 +16,7 @@ format_recipient, ) -valid_uk_phone_numbers = [ +valid_uk_mobile_phone_numbers = [ "7123456789", "07123456789", "07123 456789", @@ -44,10 +44,33 @@ ] -valid_phone_numbers = valid_uk_phone_numbers + valid_international_phone_numbers +valid_mobile_phone_numbers = valid_uk_mobile_phone_numbers + valid_international_phone_numbers + +valid_uk_landlines = [ + "0117 496 0860", # regular uk landline + "0044 117 496 0860", + "44 117 496 0860", + "+44 117 496 0860", + "016064 1234", # brampton (one digit shorter than normal) + "020 7946 0991", # london + "030 1234 5678", # non-geographic + "0500 123 4567", # corporate numbering and voip services + "0800 123 4567", # freephone + "0800 123 456", # shorter freephone + "0800 11 11", # shortest freephone + "0845 46 46", # short premium + "0900 123 4567", # premium +] +invalid_uk_landlines = [ + "0400 123 4567", # not in use + "0600 123 4567", # not in use + "0300 46 46", # short but not 01x or 08x + "0800 11 12", # short but not 01x or 08x + "0845 46 31", # short but not 01x or 08x +] -invalid_uk_phone_numbers = sum( +invalid_uk_mobile_phone_numbers = sum( [ [(phone_number, error) for phone_number in group] for error, group in [ @@ -72,14 +95,7 @@ ), ( "This does not look like a UK mobile number – double check the mobile number you entered", - ( - "08081 570364", - "+44 8081 570364", - "0117 496 0860", - "+44 117 496 0860", - "020 7946 0991", - "+44 20 7946 0991", - ), + valid_uk_landlines + invalid_uk_landlines, ), ( "Mobile numbers can only include: 0 1 2 3 4 5 6 7 8 9 ( ) + -", @@ -99,16 +115,16 @@ ) -invalid_phone_numbers = list( +invalid_mobile_phone_numbers = list( filter( lambda number: number[0] not in { "712345678910", # Could be Russia }, - invalid_uk_phone_numbers, + invalid_uk_mobile_phone_numbers, ) ) + [ - ("800000000000", "Country code not found - double check the mobile number you entered"), + ("80000000000", "Country code not found - double check the mobile number you entered"), ("1234567", "Mobile number is too short"), ("+682 1234", "Mobile number is too short"), # Cook Islands phone numbers can be 5 digits ("+12345 12345 12345 6", "Mobile number is too long"), @@ -120,7 +136,7 @@ def test_detect_international_phone_numbers(phone_number): assert is_uk_phone_number(phone_number) is False -@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) +@pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) def test_detect_uk_phone_numbers(phone_number): assert is_uk_phone_number(phone_number) is True @@ -248,7 +264,7 @@ def test_get_international_info_raises(phone_number): assert str(error.value) == "Country code not found - double check the mobile number you entered" -@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) +@pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) @pytest.mark.parametrize( "extra_args", [ @@ -263,7 +279,7 @@ def test_phone_number_accepts_valid_values(extra_args, phone_number): pytest.fail("Unexpected InvalidPhoneError") -@pytest.mark.parametrize("phone_number", valid_phone_numbers) +@pytest.mark.parametrize("phone_number", valid_mobile_phone_numbers) def test_phone_number_accepts_valid_international_values(phone_number): try: validate_phone_number(phone_number, international=True) @@ -271,7 +287,7 @@ def test_phone_number_accepts_valid_international_values(phone_number): pytest.fail("Unexpected InvalidPhoneError") -@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) +@pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) def test_valid_uk_phone_number_can_be_formatted_consistently(phone_number): assert validate_and_format_phone_number(phone_number) == "447123456789" @@ -291,7 +307,7 @@ def test_valid_international_phone_number_can_be_formatted_consistently(phone_nu assert validate_and_format_phone_number(phone_number, international=True) == expected_formatted -@pytest.mark.parametrize("phone_number, error_message", invalid_uk_phone_numbers) +@pytest.mark.parametrize("phone_number, error_message", invalid_uk_mobile_phone_numbers) @pytest.mark.parametrize( "extra_args", [ @@ -305,14 +321,14 @@ def test_phone_number_rejects_invalid_values(extra_args, phone_number, error_mes assert error_message == str(e.value) -@pytest.mark.parametrize("phone_number, error_message", invalid_phone_numbers) +@pytest.mark.parametrize("phone_number, error_message", invalid_mobile_phone_numbers) def test_phone_number_rejects_invalid_international_values(phone_number, error_message): with pytest.raises(InvalidPhoneError) as e: validate_phone_number(phone_number, international=True) assert error_message == str(e.value) -@pytest.mark.parametrize("phone_number", valid_uk_phone_numbers) +@pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) def test_validates_against_guestlist_of_phone_numbers(phone_number): assert allowed_to_send_to(phone_number, ["07123456789", "07700900460", "test@example.com"]) assert not allowed_to_send_to(phone_number, ["07700900460", "07700900461", "test@example.com"]) From bb8100918b124cf05c51f1f9d055e56bd84a06b9 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 12 Jun 2024 12:12:05 +0100 Subject: [PATCH 072/211] Move InvalidPhoneError messages to a dict within the exception obj trialling a new more explicit way to manage our error strings. We have issues managing our error messages, where we've got a bunch of highly tailored messages that were initially designed for showing errors on the frontend, but are also sent via the API. We know that some teams are checking these error message strings for specific content as they don't have any other way to refer to specific errors that they might want to bubble up on their own frontends etc. This work changes that: * Require a `code` field when creating/raising an `InvalidPhoneError`. * Define all error messages in `InvalidPhoneError.ERROR_MESSAGES`. This should be the source of truth. If a specific place needs error messages, then it probably makes sense to create a separate dict in InvalidPhoneError so that all the varying content is in one location. * If the code doesn't have a matching error message, then a KeyError will be raised, so pay attention to your exceptions carefully to make sure it's set up correctly. We may end up with helper functions or additional error message mapping dicts (for example, to handle frontend vs api error messages, international vs uk-only, landline vs mobile, a user's own number vs a recipient number). For now, lets try and keep all these in `InvalidPhoneError`. This will make it easier to reason about what error messages there are so we can be sure that they're all kept as up-to-date as possible --- .../recipient_validation/errors.py | 33 +++++++++++++++++-- .../recipient_validation/phone_number.py | 16 ++++----- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/notifications_utils/recipient_validation/errors.py b/notifications_utils/recipient_validation/errors.py index 0b9cb12ff..1a0f0b170 100644 --- a/notifications_utils/recipient_validation/errors.py +++ b/notifications_utils/recipient_validation/errors.py @@ -1,7 +1,10 @@ +from enum import StrEnum, auto + + class InvalidRecipientError(Exception): message = "Not a valid recipient address" - def __init__(self, message=None): + def __init__(self, message: str = None): super().__init__(message or self.message) @@ -10,7 +13,33 @@ class InvalidEmailError(InvalidRecipientError): class InvalidPhoneError(InvalidRecipientError): - message = "Not a valid phone number" + class Codes(StrEnum): + INVALID_NUMBER = auto() + TOO_LONG = auto() + TOO_SHORT = auto() + NOT_A_UK_MOBILE = auto() + UNKNOWN_CHARACTER = auto() + UNSUPPORTED_COUNTRY_CODE = auto() + + # TODO: Move this somewhere else maybe? Maybe not? + ERROR_MESSAGES = { + Codes.INVALID_NUMBER: "Not a valid phone number", + Codes.TOO_LONG: "Mobile number is too long", + Codes.TOO_SHORT: "Mobile number is too short", + Codes.NOT_A_UK_MOBILE: ( + "This does not look like a UK mobile number – double check the mobile number you entered" + ), + Codes.UNKNOWN_CHARACTER: "Mobile numbers can only include: 0 1 2 3 4 5 6 7 8 9 ( ) + -", + Codes.UNSUPPORTED_COUNTRY_CODE: "Country code not found - double check the mobile number you entered", + } + + def __init__(self, *, code: Codes = Codes.INVALID_NUMBER): + """ + Create an InvalidPhoneError. The code must be present in InvalidPhoneError.ERROR_MESSAGES or this will raise a + KeyError which you may not expect! + """ + self.code = code + super().__init__(message=self.ERROR_MESSAGES[code]) class InvalidAddressError(InvalidRecipientError): diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 4354dcc25..784d5d41c 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -33,7 +33,7 @@ def normalise_phone_number(number): try: list(map(int, number)) except ValueError as e: - raise InvalidPhoneError("Mobile numbers can only include: 0 1 2 3 4 5 6 7 8 9 ( ) + -") from e + raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNKNOWN_CHARACTER) from e return number.lstrip("0") @@ -90,15 +90,13 @@ def validate_uk_phone_number(number): number = normalise_phone_number(number).lstrip(UK_PREFIX).lstrip("0") if not number.startswith("7"): - raise InvalidPhoneError( - "This does not look like a UK mobile number – double check the mobile number you entered" - ) + raise InvalidPhoneError(code=InvalidPhoneError.Codes.NOT_A_UK_MOBILE) if len(number) > 10: - raise InvalidPhoneError("Mobile number is too long") + raise InvalidPhoneError(code=InvalidPhoneError.Codes.TOO_LONG) if len(number) < 10: - raise InvalidPhoneError("Mobile number is too short") + raise InvalidPhoneError(code=InvalidPhoneError.Codes.TOO_SHORT) return f"{UK_PREFIX}{number}" @@ -110,13 +108,13 @@ def validate_phone_number(number, international=False): number = normalise_phone_number(number) if len(number) < 8: - raise InvalidPhoneError("Mobile number is too short") + raise InvalidPhoneError(code=InvalidPhoneError.Codes.TOO_SHORT) if len(number) > 15: - raise InvalidPhoneError("Mobile number is too long") + raise InvalidPhoneError(code=InvalidPhoneError.Codes.TOO_LONG) if get_international_prefix(number) is None: - raise InvalidPhoneError("Country code not found - double check the mobile number you entered") + raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE) return number From ee656d3074c1667fb64568da4034d40e945635b6 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 12 Jun 2024 13:44:40 +0100 Subject: [PATCH 073/211] dont test for content in unittests --- .../recipient_validation/test_phone_number.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index 74144140b..7767ae1c8 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -75,7 +75,7 @@ [(phone_number, error) for phone_number in group] for error, group in [ ( - "Mobile number is too long", + InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.TOO_LONG], ( "712345678910", "0712345678910", @@ -85,7 +85,7 @@ ), ), ( - "Mobile number is too short", + InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.TOO_SHORT], ( "0712345678", "004471234567", @@ -94,11 +94,11 @@ ), ), ( - "This does not look like a UK mobile number – double check the mobile number you entered", + InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.NOT_A_UK_MOBILE], valid_uk_landlines + invalid_uk_landlines, ), ( - "Mobile numbers can only include: 0 1 2 3 4 5 6 7 8 9 ( ) + -", + InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.UNKNOWN_CHARACTER], ( "07890x32109", "07123 456789...", @@ -124,10 +124,13 @@ invalid_uk_mobile_phone_numbers, ) ) + [ - ("80000000000", "Country code not found - double check the mobile number you entered"), - ("1234567", "Mobile number is too short"), - ("+682 1234", "Mobile number is too short"), # Cook Islands phone numbers can be 5 digits - ("+12345 12345 12345 6", "Mobile number is too long"), + ("80000000000", InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE]), + ("1234567", InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.TOO_SHORT]), + ( + "+682 1234", + InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.TOO_SHORT], + ), # Cook Islands phone numbers can be 5 digits + ("+12345 12345 12345 6", InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.TOO_LONG]), ] @@ -261,7 +264,7 @@ def test_normalise_phone_number_raises_if_unparseable_characters(phone_number): def test_get_international_info_raises(phone_number): with pytest.raises(InvalidPhoneError) as error: get_international_phone_info(phone_number) - assert str(error.value) == "Country code not found - double check the mobile number you entered" + assert str(error.value) == InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE] @pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) From af4b88c44994f9c14a973f8a694d0167f3ed5158 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 12 Jun 2024 14:59:52 +0100 Subject: [PATCH 074/211] add legacy v2 api error messages pulled from the api repo[^1]. Keep them here near the other error messages so it's easier to compare and contrast them. [^1]: https://github.com/alphagov/notifications-api/blob/ca3bf1274dae226a17542b364b21f0662843fc4c/app/constants.py#L331-L337 --- notifications_utils/recipient_validation/errors.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/notifications_utils/recipient_validation/errors.py b/notifications_utils/recipient_validation/errors.py index 1a0f0b170..b405cdc10 100644 --- a/notifications_utils/recipient_validation/errors.py +++ b/notifications_utils/recipient_validation/errors.py @@ -33,6 +33,14 @@ class Codes(StrEnum): Codes.UNSUPPORTED_COUNTRY_CODE: "Country code not found - double check the mobile number you entered", } + LEGACY_V2_API_ERROR_MESSAGES = ERROR_MESSAGES | { + Codes.TOO_LONG: "Too many digits", + Codes.TOO_SHORT: "Not enough digits", + Codes.NOT_A_UK_MOBILE: "Not a UK mobile number", + Codes.UNKNOWN_CHARACTER: "Must not contain letters or symbols", + Codes.UNSUPPORTED_COUNTRY_CODE: "Not a valid country prefix", + } + def __init__(self, *, code: Codes = Codes.INVALID_NUMBER): """ Create an InvalidPhoneError. The code must be present in InvalidPhoneError.ERROR_MESSAGES or this will raise a @@ -41,6 +49,9 @@ def __init__(self, *, code: Codes = Codes.INVALID_NUMBER): self.code = code super().__init__(message=self.ERROR_MESSAGES[code]) + def get_legacy_v2_api_error_message(self): + return self.LEGACY_V2_API_ERROR_MESSAGES[self.code] + class InvalidAddressError(InvalidRecipientError): message = "Not a valid postal address" From ec33320a4bb735ce91510d2b880087586270ef82 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 12 Jun 2024 14:59:36 +0100 Subject: [PATCH 075/211] major version bump --- CHANGELOG.md | 7 +++++++ notifications_utils/version.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75e5265f5..1d5ac554b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # CHANGELOG +## 81.0.0 + +* BREAKING CHANGE: The constructor for `notification_utils.recipient_validation.errors.InvalidPhoneError` + - When raising InvalidPhoneError, instead of supplying a custom message, you must create an error by supplying a code from the `InvalidPhoneError.Codes` enum +* `InvalidPhoneError.code` will contain this machine-readable code for an exception if you need to examine it later +* `InvalidPhoneError.get_legacy_v2_api_error_message` returns a historical error message for use on the public v2 api + ## 80.0.1 * Reduces minimum required Gunicorn version for compatibility diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 54fb685b4..ba881669a 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "80.0.1" # 5d80aecac00f9145ca76d05c62848a87 +__version__ = "81.0.0" # 151ab871231fba7feb2eb8c23c908da9 From d7765c69ec2883278c6ff1f9080fe22321a639fd Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 18 Jun 2024 18:13:22 +0100 Subject: [PATCH 076/211] rework validation test cases as we move to using a fully featured phone number library for validation we need to update some of our test cases to make sure they're valid or invalid for the right reasons also restructure the groups more so that we can easily refer to subsets of invalid numbers more easily --- .../recipient_validation/test_phone_number.py | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index 7767ae1c8..5301575b7 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -31,16 +31,18 @@ valid_international_phone_numbers = [ - "71234567890", # Russia + "+7 (8) (495) 123-45-67", # russia + "007 (8) (495) 123-45-67", # russia + "784951234567", # Russia but without a + or 00 so it looks like it could be a uk phone number "1-202-555-0104", # USA "+12025550104", # USA "0012025550104", # USA "+0012025550104", # USA - "23051234567", # Mauritius, - "+682 12345", # Cook islands - "+3312345678", - "003312345678", - "1-2345-12345-12345", # 15 digits + "230 5 2512345", # Mauritius + "+682 50 123", # Cook islands + "+33122334455", # France + "0033122334455", # France + "+43 676 111 222 333 4", # Austrian 13 digit phone numbers ] @@ -54,7 +56,7 @@ "016064 1234", # brampton (one digit shorter than normal) "020 7946 0991", # london "030 1234 5678", # non-geographic - "0500 123 4567", # corporate numbering and voip services + "0550 123 4567", # corporate numbering and voip services "0800 123 4567", # freephone "0800 123 456", # shorter freephone "0800 11 11", # shortest freephone @@ -93,10 +95,6 @@ "+44 (0)7123 456 78", ), ), - ( - InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.NOT_A_UK_MOBILE], - valid_uk_landlines + invalid_uk_landlines, - ), ( InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.UNKNOWN_CHARACTER], ( @@ -114,16 +112,7 @@ [], ) - -invalid_mobile_phone_numbers = list( - filter( - lambda number: number[0] - not in { - "712345678910", # Could be Russia - }, - invalid_uk_mobile_phone_numbers, - ) -) + [ +invalid_international_numbers = [ ("80000000000", InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE]), ("1234567", InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.TOO_SHORT]), ( @@ -134,6 +123,25 @@ ] +# NOTE: includes landlines +invalid_mobile_phone_numbers = ( + list( + filter( + lambda number: number[0] + not in { + "712345678910", # Could be Russia + }, + invalid_uk_mobile_phone_numbers, + ) + ) + + invalid_international_numbers + + [ + (num, InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.NOT_A_UK_MOBILE]) + for num in valid_uk_landlines + invalid_uk_landlines + ] +) + + @pytest.mark.parametrize("phone_number", valid_international_phone_numbers) def test_detect_international_phone_numbers(phone_number): assert is_uk_phone_number(phone_number) is False From 3fe550e64901711ad590b58bd7d16d9058bef4bc Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 12 Jun 2024 14:02:56 +0100 Subject: [PATCH 077/211] use phonenumbers to validate uk landlines we already use phonenumbers in format_phone_number_human_readable as it nicely formats numbers - but hang on, it has a whole suite of validation features too. We often find other teams asking us how we validate phone numbers so they can replicate it on their own frontend. We link to this repo but it would be much nicer if we could just point them to a well supported third party library with a variety of implementations in different languages. Start by using phonenumbers in a separate function so we can compare the test output. We could potentially run validation in parallel with our regular output to ensure it matches. I'm not too worried if things we currently fail are allowed in by phonenumbers, as the chances are they're more correct than we are - but it's important that any numbers we currently accept and deliver to also work in phonenumbers. --- .../recipient_validation/errors.py | 27 ++++++++++- .../recipient_validation/phone_number.py | 48 +++++++++++++++++++ .../recipient_validation/test_phone_number.py | 36 ++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) diff --git a/notifications_utils/recipient_validation/errors.py b/notifications_utils/recipient_validation/errors.py index b405cdc10..0dd7d9495 100644 --- a/notifications_utils/recipient_validation/errors.py +++ b/notifications_utils/recipient_validation/errors.py @@ -1,5 +1,7 @@ from enum import StrEnum, auto +import phonenumbers + class InvalidRecipientError(Exception): message = "Not a valid recipient address" @@ -23,7 +25,10 @@ class Codes(StrEnum): # TODO: Move this somewhere else maybe? Maybe not? ERROR_MESSAGES = { - Codes.INVALID_NUMBER: "Not a valid phone number", + # this catches numbers with the right length but wrong digits + # for example UK numbers cannot start "06" as that hasn't been assigned to a purpose by ofcom, + # or a 9 digit UK number that does not start "01" or "0800". + Codes.INVALID_NUMBER: "TODO: CONTENT! Number is not valid – double check the mobile number you entered", Codes.TOO_LONG: "Mobile number is too long", Codes.TOO_SHORT: "Mobile number is too short", Codes.NOT_A_UK_MOBILE: ( @@ -49,6 +54,26 @@ def __init__(self, *, code: Codes = Codes.INVALID_NUMBER): self.code = code super().__init__(message=self.ERROR_MESSAGES[code]) + @classmethod + def from_phonenumbers_validationresult(cls, reason: phonenumbers.ValidationResult) -> str: + match reason: + case phonenumbers.ValidationResult.TOO_LONG: + code = cls.Codes.TOO_LONG + # is_possible_local_only implies a number without an area code. Lets just call it too short. + case phonenumbers.ValidationResult.TOO_SHORT | phonenumbers.ValidationResult.IS_POSSIBLE_LOCAL_ONLY: + code = cls.Codes.TOO_SHORT + case phonenumbers.ValidationResult.INVALID_COUNTRY_CODE: + code = cls.Codes.UNSUPPORTED_COUNTRY_CODE + case phonenumbers.ValidationResult.IS_POSSIBLE: + raise ValueError("Cannot create InvalidPhoneNumber for ValidationResult.IS_POSSIBLE") + case phonenumbers.ValidationResult.INVALID_LENGTH: + code = cls.Codes.INVALID_NUMBER + + case _: + code = cls.Codes.INVALID_NUMBER + + return cls(code=code) + def get_legacy_v2_api_error_message(self): return self.LEGACY_V2_API_ERROR_MESSAGES[self.code] diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 784d5d41c..bded52074 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -151,3 +151,51 @@ def format_phone_number_human_readable(phone_number): else phonenumbers.PhoneNumberFormat.NATIONAL ), ) + + +class UKLandline: + @staticmethod + def raise_if_phone_number_contains_invalid_characters(number: str) -> None: + chars = set(number) + if chars - {*ALL_WHITESPACE + "()-+" + "0123456789"}: + raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNKNOWN_CHARACTER) + + @staticmethod + def validate_mobile_or_uk_landline(phone_number: str, *, allow_international: bool) -> phonenumbers.PhoneNumber: + """ + Validate a phone number and return the PhoneNumber object + + Tries best effort validation, and has some extra logic to make the validation closer to our existing validation + including: + + * Being stricter with rogue alphanumeric characters. (eg don't allow https://en.wikipedia.org/wiki/Phoneword) + * Additional parsing steps to check if there was a + or leading 0 stripped off the beginning of the number that + changes whether it is parsed as international or not. + * Convert error codes to match existing Notify error codes + """ + # notify's old validation code is stricter than phonenumbers in not allowing letters etc, so need to catch some + # of those cases separately before we parse with the phonenumbers library + UKLandline.raise_if_phone_number_contains_invalid_characters(phone_number) + + try: + # parse number as GB - if there's no country code, try and parse it as a UK number + number = phonenumbers.parse(phone_number, "GB") + except phonenumbers.NumberParseException as e: + raise InvalidPhoneError(code=InvalidPhoneError.Codes.INVALID_NUMBER) from e + + if not allow_international and str(number.country_code) != UK_PREFIX: + raise InvalidPhoneError(code=InvalidPhoneError.Codes.NOT_A_UK_MOBILE) + + if str(number.country_code) not in COUNTRY_PREFIXES + ["+44"]: + raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE) + + if (reason := phonenumbers.is_possible_number_with_reason(number)) != phonenumbers.ValidationResult.IS_POSSIBLE: + raise InvalidPhoneError.from_phonenumbers_validationresult(reason) + + if not phonenumbers.is_valid_number(number): + # is_possible just checks the length of a number for that country/region. is_valid checks if it's + # a valid sequence of numbers. This doesn't cover "is this number registered to an MNO". + # For example UK numbers cannot start "06" as that hasn't been assigned to a purpose by ofcom + raise InvalidPhoneError(code=InvalidPhoneError.Codes.INVALID_NUMBER) + + return number diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index 5301575b7..4091fa5ea 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -2,6 +2,7 @@ from notifications_utils.recipient_validation.errors import InvalidPhoneError from notifications_utils.recipient_validation.phone_number import ( + UKLandline, format_phone_number_human_readable, get_international_phone_info, international_phone_info, @@ -399,3 +400,38 @@ def test_try_format_recipient_doesnt_throw(): def test_format_phone_number_human_readable_doenst_throw(): assert format_phone_number_human_readable("ALPHANUM3R1C") == "ALPHANUM3R1C" + + +class TestUKLandlineValidation: + + @pytest.mark.parametrize("phone_number, error_message", invalid_uk_mobile_phone_numbers) + def test_rejects_invalid_uk_mobile_phone_numbers(self, phone_number, error_message): + # problem is `invalid_uk_mobile_phone_numbers` also includes valid uk landlines + with pytest.raises(InvalidPhoneError): + UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=False) + # assert e.value.code == InvalidPhoneError.Codes.INVALID_NUMBER + + @pytest.mark.parametrize("phone_number", invalid_uk_landlines) + def test_rejects_invalid_uk_landlines(self, phone_number): + with pytest.raises(InvalidPhoneError) as e: + UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=False) + assert e.value.code == InvalidPhoneError.Codes.INVALID_NUMBER + + @pytest.mark.parametrize( + "phone_number, error_message", invalid_uk_mobile_phone_numbers + invalid_international_numbers + ) + def test_rejects_invalid_international_phone_numbers(self, phone_number, error_message): + with pytest.raises(InvalidPhoneError): + UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=True) + + @pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) + def test_allows_valid_uk_mobile_phone_numbers(self, phone_number): + UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=False) + + @pytest.mark.parametrize("phone_number", valid_international_phone_numbers) + def test_allows_valid_international_phone_numbers(self, phone_number): + UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=True) + + @pytest.mark.parametrize("phone_number", valid_uk_landlines) + def test_allows_valid_uk_landlines(self, phone_number): + UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=True) From ec6fc0ed6e455e77f5e0bba19c1ccd74fbde9251 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 18 Jun 2024 23:52:16 +0100 Subject: [PATCH 078/211] add code to handle international numbers without + or 00 prefix We have seen excel interpret cells with phone numbers in as numeric fields (rather than strings). It might strip leading zeros or + prefixes as they are implied with actual countable numbers. But they're a crucial part of identifiying international phone numbers. the phonenumbers library doesn't check for this. If you pass in a number without an international prefix, it assumes you're trying to call a local (GB) number, so validates it based on that. We're more likely to be sending messages to UK numbers, so start with that, but if the number isn't a valid UK number, AND the service can send to international numbers, then we can try and parse the number again but this time with a + on the beginning to force it to identify as a foreign number. If this parsing still doesn't work, then the number is clearly invalid. But if the number works, then use that with the + as the root number that we can send to later. --- .../recipient_validation/phone_number.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index bded52074..65b9106e8 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -1,4 +1,5 @@ from collections import namedtuple +from contextlib import suppress import phonenumbers from flask import current_app @@ -190,7 +191,12 @@ def validate_mobile_or_uk_landline(phone_number: str, *, allow_international: bo raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE) if (reason := phonenumbers.is_possible_number_with_reason(number)) != phonenumbers.ValidationResult.IS_POSSIBLE: - raise InvalidPhoneError.from_phonenumbers_validationresult(reason) + if allow_international and ( + forced_international_number := UKLandline._validate_forced_international_number(phone_number) + ): + number = forced_international_number + else: + raise InvalidPhoneError.from_phonenumbers_validationresult(reason) if not phonenumbers.is_valid_number(number): # is_possible just checks the length of a number for that country/region. is_valid checks if it's @@ -199,3 +205,18 @@ def validate_mobile_or_uk_landline(phone_number: str, *, allow_international: bo raise InvalidPhoneError(code=InvalidPhoneError.Codes.INVALID_NUMBER) return number + + @staticmethod + def _validate_forced_international_number(phone_number: str) -> phonenumbers.PhoneNumber | None: + """ + phonenumbers assumes a number without a + or 00 at beginning is always a local number. Given that we know excel + sometimes strips these, if it doesn't parse as a UK number, lets try forcing it to be recognised as an + international number + """ + with suppress(phonenumbers.NumberParseException): + forced_international_number = phonenumbers.parse(f"+{phone_number}") + + if phonenumbers.is_possible_number(forced_international_number): + return forced_international_number + + return None From f4d75509d017fa74e1ed361f4eef41fea95fa130 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 18 Jun 2024 23:58:41 +0100 Subject: [PATCH 079/211] tweak a couple of test cases: * +800 is actually a valid country code, +801 isn't * phonenumbers can't parse \ufeff in the middle of strings. it's fine with it at the end. i think this is good enough for a first pass --- tests/recipient_validation/test_phone_number.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index 4091fa5ea..f0e3ac2e2 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -27,7 +27,7 @@ "+447123456789", "+44 7123 456 789", "+44 (0)7123 456 789", - "\u200b\t\t+44 (0)7123 \ufeff 456 789 \r\n", + "\u200b\t\t+44 (0)7123 456 789\ufeff \r\n", ] @@ -114,7 +114,7 @@ ) invalid_international_numbers = [ - ("80000000000", InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE]), + ("80100000000", InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE]), ("1234567", InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.TOO_SHORT]), ( "+682 1234", From 34fed3729a204ddeb688e02c1e96aeba540d3493 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 1 Jul 2024 15:15:46 +0100 Subject: [PATCH 080/211] change UKLandline to PhoneNumber and make it instanced previously, UKLandline was a "namespace" style class that just contained a set of static methods instead, make it an instanced class where the constructor parses and validates the phone number. This'll make the interface cleaner for if we want to, eg, return the number in different formats, or extend the class to handle different cases (eg landline vs not, international vs not, strict validation vs not) later. --- .../recipient_validation/phone_number.py | 28 +++++++++++++------ .../recipient_validation/test_phone_number.py | 16 +++++------ 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 65b9106e8..1c872b88b 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -154,15 +154,27 @@ def format_phone_number_human_readable(phone_number): ) -class UKLandline: +class PhoneNumber: + """ + A class that contains phone number validation. + + Supports mobile and landline numbers. When creating an object you must specify whether you are expecting + international phone numbers to be allowed or not. + """ + + def __init__(self, phone_number: str, *, allow_international: bool) -> None: + self.raw_input = phone_number + self.allow_international = allow_international + self.number = self.validate_phone_number(phone_number) + self.prefix = str(self.number.country_code) + @staticmethod - def raise_if_phone_number_contains_invalid_characters(number: str) -> None: + def _raise_if_phone_number_contains_invalid_characters(number: str) -> None: chars = set(number) if chars - {*ALL_WHITESPACE + "()-+" + "0123456789"}: raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNKNOWN_CHARACTER) - @staticmethod - def validate_mobile_or_uk_landline(phone_number: str, *, allow_international: bool) -> phonenumbers.PhoneNumber: + def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: """ Validate a phone number and return the PhoneNumber object @@ -176,7 +188,7 @@ def validate_mobile_or_uk_landline(phone_number: str, *, allow_international: bo """ # notify's old validation code is stricter than phonenumbers in not allowing letters etc, so need to catch some # of those cases separately before we parse with the phonenumbers library - UKLandline.raise_if_phone_number_contains_invalid_characters(phone_number) + self._raise_if_phone_number_contains_invalid_characters(phone_number) try: # parse number as GB - if there's no country code, try and parse it as a UK number @@ -184,15 +196,15 @@ def validate_mobile_or_uk_landline(phone_number: str, *, allow_international: bo except phonenumbers.NumberParseException as e: raise InvalidPhoneError(code=InvalidPhoneError.Codes.INVALID_NUMBER) from e - if not allow_international and str(number.country_code) != UK_PREFIX: + if not self.allow_international and str(number.country_code) != UK_PREFIX: raise InvalidPhoneError(code=InvalidPhoneError.Codes.NOT_A_UK_MOBILE) if str(number.country_code) not in COUNTRY_PREFIXES + ["+44"]: raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE) if (reason := phonenumbers.is_possible_number_with_reason(number)) != phonenumbers.ValidationResult.IS_POSSIBLE: - if allow_international and ( - forced_international_number := UKLandline._validate_forced_international_number(phone_number) + if self.allow_international and ( + forced_international_number := self._validate_forced_international_number(phone_number) ): number = forced_international_number else: diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index f0e3ac2e2..522b9f484 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -2,7 +2,7 @@ from notifications_utils.recipient_validation.errors import InvalidPhoneError from notifications_utils.recipient_validation.phone_number import ( - UKLandline, + PhoneNumber, format_phone_number_human_readable, get_international_phone_info, international_phone_info, @@ -402,19 +402,19 @@ def test_format_phone_number_human_readable_doenst_throw(): assert format_phone_number_human_readable("ALPHANUM3R1C") == "ALPHANUM3R1C" -class TestUKLandlineValidation: +class TestPhoneNumbeClass: @pytest.mark.parametrize("phone_number, error_message", invalid_uk_mobile_phone_numbers) def test_rejects_invalid_uk_mobile_phone_numbers(self, phone_number, error_message): # problem is `invalid_uk_mobile_phone_numbers` also includes valid uk landlines with pytest.raises(InvalidPhoneError): - UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=False) + PhoneNumber(phone_number, allow_international=False) # assert e.value.code == InvalidPhoneError.Codes.INVALID_NUMBER @pytest.mark.parametrize("phone_number", invalid_uk_landlines) def test_rejects_invalid_uk_landlines(self, phone_number): with pytest.raises(InvalidPhoneError) as e: - UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=False) + PhoneNumber(phone_number, allow_international=False) assert e.value.code == InvalidPhoneError.Codes.INVALID_NUMBER @pytest.mark.parametrize( @@ -422,16 +422,16 @@ def test_rejects_invalid_uk_landlines(self, phone_number): ) def test_rejects_invalid_international_phone_numbers(self, phone_number, error_message): with pytest.raises(InvalidPhoneError): - UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=True) + PhoneNumber(phone_number, allow_international=True) @pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) def test_allows_valid_uk_mobile_phone_numbers(self, phone_number): - UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=False) + PhoneNumber(phone_number, allow_international=False) @pytest.mark.parametrize("phone_number", valid_international_phone_numbers) def test_allows_valid_international_phone_numbers(self, phone_number): - UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=True) + PhoneNumber(phone_number, allow_international=True) @pytest.mark.parametrize("phone_number", valid_uk_landlines) def test_allows_valid_uk_landlines(self, phone_number): - UKLandline.validate_mobile_or_uk_landline(phone_number, allow_international=True) + PhoneNumber(phone_number, allow_international=True) From fc2a2a41fba5bc40404bb3abd629a68f9e235aaf Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 1 Jul 2024 17:23:21 +0100 Subject: [PATCH 081/211] add additional helper methods to the PhoneNumber class These are all copies of existing functionality, but moved to operate on an already created and validated PhoneNumber object rather than a raw user input string. is_uk_phone_number: returns if it's a UK phone number. includes crown dependencies like jersey, guernsey, isle of man etc. get_international_phone_info returns international info that we use for billing information etc. should_use_numeric_sender returns if the recipient country requires us to use a notify-wide numeric sender rather than the service's intended sender. get_normalised_format returns an E.164 compliant phone number - ie: with a country code, and no spaces/dashes/brackets or other human readability features get_human_readable_format returns a human readable format. Specific to the country the number is for. Note that the test cases have moved around a bit but all the test cases used here are re-used from the existing tests for the old functions --- .../recipient_validation/phone_number.py | 53 ++++ .../recipient_validation/test_phone_number.py | 230 +++++++++++------- 2 files changed, 194 insertions(+), 89 deletions(-) diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 1c872b88b..2a25cc966 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -232,3 +232,56 @@ def _validate_forced_international_number(phone_number: str) -> phonenumbers.Pho return forced_international_number return None + + def is_uk_phone_number(self): + """ + Returns if the number's country code is +44 - note, this includes any regions that also use the +44 country code + such as isle of man/jersey/guernsey. You can distinguish these with `number._is_a_crown_dependency_number` + """ + return self.number.country_code == 44 + + def get_international_phone_info(self): + return international_phone_info( + international=not self.is_uk_phone_number(), + crown_dependency=self._is_a_crown_dependency_number(), + country_prefix=self.prefix, + billable_units=INTERNATIONAL_BILLING_RATES[self.prefix]["billable_units"], + ) + + def _is_a_crown_dependency_number(self): + """ + Returns True for phone numbers from Jersey, Guernsey, Isle of Man, etc + """ + return self.is_uk_phone_number() and phonenumbers.region_code_for_number(self.number) != "GB" + + def should_use_numeric_sender(self): + """ + Some countries need a specific sender to be used rather than whatever the service has specified + """ + return INTERNATIONAL_BILLING_RATES[self.prefix]["attributes"]["alpha"] == "NO" + + def get_normalised_format(self): + return str(self) + + def __str__(self): + """ + Returns a normalised phone number including international country code suitable to send to providers + """ + formatted = phonenumbers.format_number(self.number, phonenumbers.PhoneNumberFormat.E164) + # TODO: If our suppliers let us send the plus, then we should do so, for consistency/accuracy. + if self.is_uk_phone_number(): + # if it's a UK number we strip the + and just send eg "447700900100" + return formatted[1:] + else: + return formatted + + def get_human_readable_format(self): + # comparable to `format_phone_number_human_readable` + return phonenumbers.format_number( + self.number, + ( + phonenumbers.PhoneNumberFormat.INTERNATIONAL + if self.number.country_code != 44 + else phonenumbers.PhoneNumberFormat.NATIONAL + ), + ) diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index 522b9f484..0c6cc1cd4 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -143,6 +143,91 @@ ) +international_phone_info_fixtures = [ + ( + "07900900123", + international_phone_info( + international=False, + crown_dependency=False, + country_prefix="44", # UK + billable_units=1, + ), + ), + ( + "07700900123", + international_phone_info( + international=False, + crown_dependency=False, + country_prefix="44", # Number in TV range + billable_units=1, + ), + ), + ( + "07700800123", + international_phone_info( + international=True, + crown_dependency=True, + country_prefix="44", # UK Crown dependency, so prefix same as UK + billable_units=1, + ), + ), + ( + "20-12-1234-1234", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="20", # Egypt + billable_units=3, + ), + ), + ( + "00201212341234", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="20", # Egypt + billable_units=3, + ), + ), + ( + "1664000000000", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="1664", # Montserrat + billable_units=3, + ), + ), + ( + "71234567890", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="7", # Russia + billable_units=4, + ), + ), + ( + "1-202-555-0104", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="1", # USA + billable_units=1, + ), + ), + ( + "+23051234567", + international_phone_info( + international=True, + crown_dependency=False, + country_prefix="230", # Mauritius + billable_units=2, + ), + ), +] + + @pytest.mark.parametrize("phone_number", valid_international_phone_numbers) def test_detect_international_phone_numbers(phone_number): assert is_uk_phone_number(phone_number) is False @@ -153,92 +238,7 @@ def test_detect_uk_phone_numbers(phone_number): assert is_uk_phone_number(phone_number) is True -@pytest.mark.parametrize( - "phone_number, expected_info", - [ - ( - "07900900123", - international_phone_info( - international=False, - crown_dependency=False, - country_prefix="44", # UK - billable_units=1, - ), - ), - ( - "07700900123", - international_phone_info( - international=False, - crown_dependency=False, - country_prefix="44", # Number in TV range - billable_units=1, - ), - ), - ( - "07700800123", - international_phone_info( - international=True, - crown_dependency=True, - country_prefix="44", # UK Crown dependency, so prefix same as UK - billable_units=1, - ), - ), - ( - "20-12-1234-1234", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="20", # Egypt - billable_units=3, - ), - ), - ( - "00201212341234", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="20", # Egypt - billable_units=3, - ), - ), - ( - "1664000000000", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="1664", # Montserrat - billable_units=3, - ), - ), - ( - "71234567890", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="7", # Russia - billable_units=4, - ), - ), - ( - "1-202-555-0104", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="1", # USA - billable_units=1, - ), - ), - ( - "+23051234567", - international_phone_info( - international=True, - crown_dependency=False, - country_prefix="230", # Mauritius - billable_units=2, - ), - ), - ], -) +@pytest.mark.parametrize("phone_number, expected_info", international_phone_info_fixtures) def test_get_international_info(phone_number, expected_info): assert get_international_phone_info(phone_number) == expected_info @@ -426,12 +426,64 @@ def test_rejects_invalid_international_phone_numbers(self, phone_number, error_m @pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) def test_allows_valid_uk_mobile_phone_numbers(self, phone_number): - PhoneNumber(phone_number, allow_international=False) + assert PhoneNumber(phone_number, allow_international=False).is_uk_phone_number() is True @pytest.mark.parametrize("phone_number", valid_international_phone_numbers) def test_allows_valid_international_phone_numbers(self, phone_number): - PhoneNumber(phone_number, allow_international=True) + assert PhoneNumber(phone_number, allow_international=True).is_uk_phone_number() is False @pytest.mark.parametrize("phone_number", valid_uk_landlines) def test_allows_valid_uk_landlines(self, phone_number): - PhoneNumber(phone_number, allow_international=True) + assert PhoneNumber(phone_number, allow_international=True).is_uk_phone_number() is True + + @pytest.mark.parametrize("phone_number, expected_info", international_phone_info_fixtures) + def test_get_international_phone_info(self, phone_number, expected_info): + assert PhoneNumber(phone_number, allow_international=True).get_international_phone_info() == expected_info + + @pytest.mark.parametrize( + "number, expected", + [ + ("48123654789", False), # Poland alpha: Yes + ("1-403-123-5687", True), # Canada alpha: No + ("40123548897", False), # Romania alpha: REG + ("+60123451345", True), # Malaysia alpha: NO + ], + ) + def test_should_use_numeric_sender(self, number, expected): + assert PhoneNumber(number, allow_international=True).should_use_numeric_sender() == expected + + @pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) + def test_get_normalised_format_works_for_uk_mobiles(self, phone_number): + assert PhoneNumber(phone_number, allow_international=True).get_normalised_format() == "447123456789" + + @pytest.mark.parametrize( + "phone_number, expected_formatted", + [ + ("71234567890", "71234567890"), + ("1-202-555-0104", "12025550104"), + ("+12025550104", "12025550104"), + ("0012025550104", "12025550104"), + ("+0012025550104", "12025550104"), + ("23051234567", "23051234567"), + ], + ) + def test_get_normalised_format_works_for_international_numbers(self, phone_number, expected_formatted): + assert PhoneNumber(phone_number, allow_international=True).get_normalised_format() == expected_formatted + + @pytest.mark.parametrize( + "phone_number, expected_formatted", + [ + ("07900900123", "07900 900123"), # UK + ("+44(0)7900900123", "07900 900123"), # UK + ("447900900123", "07900 900123"), # UK + ("20-12-1234-1234", "+20 12 12341234"), # Egypt + ("00201212341234", "+20 12 12341234"), # Egypt + ("1664 0000000", "+1 664-000-0000"), # Montserrat + ("7 499 1231212", "+7 499 123-12-12"), # Moscow (Russia) + ("1-202-555-0104", "+1 202-555-0104"), # Washington DC (USA) + ("+23051234567", "+230 5123 4567"), # Mauritius + ("33(0)1 12345678", "+33 1 12 34 56 78"), # Paris (France) + ], + ) + def test_get_human_readable_format(self, phone_number, expected_formatted): + assert PhoneNumber(phone_number, allow_international=True).get_human_readable_format() == expected_formatted From 948abee251ed66e2a7f40fd69ddd01a22fa5c300 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 1 Jul 2024 22:54:01 +0100 Subject: [PATCH 082/211] update helper functions to handle notify quirks * notify considers some NAPN (north american numbering plan - all the countries that share a +1 dialling code) countries separately. One example of this is montserrat, +1 664. phonenumbers returns just +1 for the country code, but we need to look them up in our yml file via the combined country and area code. So we need to do a bit of parsing of the returned phone number in that case to pull out those extra digits, but only if that four digit +1xxx combo is found in our international billing rates, otherwise it might just be eg a US area code like 212. * strip the leading + from our formatted numbers. Our formatted numbers do not quite comply with the E.164 standard for displaying phone numbers - when we pass to our sms providers we strip the +. It would be nice to not strip this plus sign and instead send the full number but for now lets maintain consistency --- .../recipient_validation/phone_number.py | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 2a25cc966..45de43493 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -166,7 +166,6 @@ def __init__(self, phone_number: str, *, allow_international: bool) -> None: self.raw_input = phone_number self.allow_international = allow_international self.number = self.validate_phone_number(phone_number) - self.prefix = str(self.number.country_code) @staticmethod def _raise_if_phone_number_contains_invalid_characters(number: str) -> None: @@ -233,16 +232,35 @@ def _validate_forced_international_number(phone_number: str) -> phonenumbers.Pho return None + @property + def prefix(self): + """ + Returns the international dialing code for looking up data in our international_billing_rates.yml file + + in our billing rates yml file, countries in the North American Numbering Plan (+1) may fall under + US/Canada/Dominican Republic (just +1) or they may have their own specific area code within the plan, eg + Montserrat with numbers like "+1 664 xxx xxxx". This means we need to check the area code first to see if + it's a regular area code or a full country code. + """ + if self.number.country_code == 1: + country_and_area_code = phonenumbers.format_number(self.number, phonenumbers.PhoneNumberFormat.E164)[1:5] + if country_and_area_code in INTERNATIONAL_BILLING_RATES: + return country_and_area_code + return str(self.number.country_code) + def is_uk_phone_number(self): """ - Returns if the number's country code is +44 - note, this includes any regions that also use the +44 country code - such as isle of man/jersey/guernsey. You can distinguish these with `number._is_a_crown_dependency_number` + Returns if the number starts with +44. Note, this includes international numbers for crown dependencies such as + jersey/guernsey. + + # TODO: check if we still need this - looking at api, this might be able to be removed entirely since it's + # always used in conjunction with should_use_numeric_sender """ return self.number.country_code == 44 def get_international_phone_info(self): return international_phone_info( - international=not self.is_uk_phone_number(), + international=phonenumbers.region_code_for_number(self.number) != "GB", crown_dependency=self._is_a_crown_dependency_number(), country_prefix=self.prefix, billable_units=INTERNATIONAL_BILLING_RATES[self.prefix]["billable_units"], @@ -268,12 +286,9 @@ def __str__(self): Returns a normalised phone number including international country code suitable to send to providers """ formatted = phonenumbers.format_number(self.number, phonenumbers.PhoneNumberFormat.E164) + # strip the plus and just pass numbers to our suppliers. # TODO: If our suppliers let us send the plus, then we should do so, for consistency/accuracy. - if self.is_uk_phone_number(): - # if it's a UK number we strip the + and just send eg "447700900100" - return formatted[1:] - else: - return formatted + return formatted[1:] def get_human_readable_format(self): # comparable to `format_phone_number_human_readable` From 450c30c7464cf34d1482d45904f2829c35536b71 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 1 Jul 2024 23:13:28 +0100 Subject: [PATCH 083/211] Update test cases to use semantically valid phone numbers Lots of the UK phone number tests used "07123456789". This isn't actually a valid number, as the "0712" first part of the phone number hasn't been assigned to any carriers by ofcom[^1]. Update these to one that works just to get the tests to pass. Note this also means that numbers reserved for TV usage (07700900xxx) are no longer valid! This may have an impact on our users and perhaps on our functional tests if we expect these to be delivered - as with the new code logic they would now fail validation as they're accurately not valid phone numbers. [^1]: https://github.com/google/libphonenumber/blob/master/resources/carrier/en/44.txt --- .../recipient_validation/test_phone_number.py | 93 +++++++++---------- 1 file changed, 42 insertions(+), 51 deletions(-) diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index 0c6cc1cd4..de1e2dd76 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -18,16 +18,16 @@ ) valid_uk_mobile_phone_numbers = [ - "7123456789", - "07123456789", - "07123 456789", - "07123-456-789", - "00447123456789", - "00 44 7123456789", - "+447123456789", - "+44 7123 456 789", - "+44 (0)7123 456 789", - "\u200b\t\t+44 (0)7123 456 789\ufeff \r\n", + "7723456789", + "07723456789", + "07723 456789", + "07723-456-789", + "00447723456789", + "00 44 7723456789", + "+447723456789", + "+44 7723 456 789", + "+44 (0)7723 456 789", + "\u200b\t\t+44 (0)7723 456 789\ufeff \r\n", ] @@ -80,29 +80,29 @@ ( InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.TOO_LONG], ( - "712345678910", - "0712345678910", - "0044712345678910", - "0044712345678910", - "+44 (0)7123 456 789 10", + "772345678910", + "0772345678910", + "0044772345678910", + "0044772345678910", + "+44 (0)7723 456 789 10", ), ), ( InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.TOO_SHORT], ( - "0712345678", - "004471234567", - "00447123456", - "+44 (0)7123 456 78", + "0772345678", + "004477234567", + "00447723456", + "+44 (0)7723 456 78", ), ), ( InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.UNKNOWN_CHARACTER], ( "07890x32109", - "07123 456789...", - "07123 ☟☜⬇⬆☞☝", - "07123☟☜⬇⬆☞☝", + "07723 456789...", + "07723 ☟☜⬇⬆☞☝", + "07723☟☜⬇⬆☞☝", '07";DROP TABLE;"', "+44 07ab cde fgh", "ALPHANUM3R1C", @@ -130,7 +130,7 @@ filter( lambda number: number[0] not in { - "712345678910", # Could be Russia + "772345678910", # Could be Russia }, invalid_uk_mobile_phone_numbers, ) @@ -145,7 +145,7 @@ international_phone_info_fixtures = [ ( - "07900900123", + "07723456789", international_phone_info( international=False, crown_dependency=False, @@ -154,16 +154,7 @@ ), ), ( - "07700900123", - international_phone_info( - international=False, - crown_dependency=False, - country_prefix="44", # Number in TV range - billable_units=1, - ), - ), - ( - "07700800123", + "07797800123", international_phone_info( international=True, crown_dependency=True, @@ -190,7 +181,7 @@ ), ), ( - "1664000000000", + "16644913789", international_phone_info( international=True, crown_dependency=False, @@ -199,7 +190,7 @@ ), ), ( - "71234567890", + "77234567890", international_phone_info( international=True, crown_dependency=False, @@ -217,7 +208,7 @@ ), ), ( - "+23051234567", + "+23052512345", international_phone_info( international=True, crown_dependency=False, @@ -301,18 +292,18 @@ def test_phone_number_accepts_valid_international_values(phone_number): @pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) def test_valid_uk_phone_number_can_be_formatted_consistently(phone_number): - assert validate_and_format_phone_number(phone_number) == "447123456789" + assert validate_and_format_phone_number(phone_number) == "447723456789" @pytest.mark.parametrize( "phone_number, expected_formatted", [ - ("71234567890", "71234567890"), + ("77234567890", "77234567890"), ("1-202-555-0104", "12025550104"), ("+12025550104", "12025550104"), ("0012025550104", "12025550104"), ("+0012025550104", "12025550104"), - ("23051234567", "23051234567"), + ("23052512345", "23052512345"), ], ) def test_valid_international_phone_number_can_be_formatted_consistently(phone_number, expected_formatted): @@ -342,7 +333,7 @@ def test_phone_number_rejects_invalid_international_values(phone_number, error_m @pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) def test_validates_against_guestlist_of_phone_numbers(phone_number): - assert allowed_to_send_to(phone_number, ["07123456789", "07700900460", "test@example.com"]) + assert allowed_to_send_to(phone_number, ["07723456789", "07700900460", "test@example.com"]) assert not allowed_to_send_to(phone_number, ["07700900460", "07700900461", "test@example.com"]) @@ -444,8 +435,8 @@ def test_get_international_phone_info(self, phone_number, expected_info): "number, expected", [ ("48123654789", False), # Poland alpha: Yes - ("1-403-123-5687", True), # Canada alpha: No - ("40123548897", False), # Romania alpha: REG + ("1-403-555-0104", True), # Canada alpha: No + ("40 21 201 7200", False), # Romania alpha: REG ("+60123451345", True), # Malaysia alpha: NO ], ) @@ -454,17 +445,17 @@ def test_should_use_numeric_sender(self, number, expected): @pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) def test_get_normalised_format_works_for_uk_mobiles(self, phone_number): - assert PhoneNumber(phone_number, allow_international=True).get_normalised_format() == "447123456789" + assert PhoneNumber(phone_number, allow_international=True).get_normalised_format() == "447723456789" @pytest.mark.parametrize( "phone_number, expected_formatted", [ - ("71234567890", "71234567890"), + ("74991231212", "74991231212"), ("1-202-555-0104", "12025550104"), ("+12025550104", "12025550104"), ("0012025550104", "12025550104"), ("+0012025550104", "12025550104"), - ("23051234567", "23051234567"), + ("23052512345", "23052512345"), ], ) def test_get_normalised_format_works_for_international_numbers(self, phone_number, expected_formatted): @@ -473,15 +464,15 @@ def test_get_normalised_format_works_for_international_numbers(self, phone_numbe @pytest.mark.parametrize( "phone_number, expected_formatted", [ - ("07900900123", "07900 900123"), # UK - ("+44(0)7900900123", "07900 900123"), # UK - ("447900900123", "07900 900123"), # UK + ("07723456789", "07723 456789"), # UK + ("+44(0)7723456789", "07723 456789"), # UK + ("447723456789", "07723 456789"), # UK ("20-12-1234-1234", "+20 12 12341234"), # Egypt ("00201212341234", "+20 12 12341234"), # Egypt - ("1664 0000000", "+1 664-000-0000"), # Montserrat + ("1664 491 3789", "+1 664-491-3789"), # Montserrat ("7 499 1231212", "+7 499 123-12-12"), # Moscow (Russia) ("1-202-555-0104", "+1 202-555-0104"), # Washington DC (USA) - ("+23051234567", "+230 5123 4567"), # Mauritius + ("+23052512345", "+230 5251 2345"), # Mauritius ("33(0)1 12345678", "+33 1 12 34 56 78"), # Paris (France) ], ) From 9311e34b4aa6e0f18ae7f2db40b845071c987aba Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 3 Jul 2024 12:33:36 +0100 Subject: [PATCH 084/211] normalise and retry phone numbers if validation fails From a million phone numbers validated succesfully with our old code, approximately 3.5% of these do not validate with the new phonenumbers based code. Start the framework for doing some extra normalisation for these numbers if they are formatted in slightly weird ways that we think we can make an attempt to understand/decipher. Some of these 3.5% will still fail, because our new validation is stricter (as it knows about area codes within countries etc) --- .../recipient_validation/phone_number.py | 33 ++++++++++++++++++- .../recipient_validation/test_phone_number.py | 9 ++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 45de43493..3fd814979 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -202,7 +202,9 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE) if (reason := phonenumbers.is_possible_number_with_reason(number)) != phonenumbers.ValidationResult.IS_POSSIBLE: - if self.allow_international and ( + if normalised_number := self._validate_aggressively_normalised_number(phone_number): + number = normalised_number + elif self.allow_international and ( forced_international_number := self._validate_forced_international_number(phone_number) ): number = forced_international_number @@ -217,6 +219,35 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: return number + @staticmethod + def _validate_aggressively_normalised_number(phone_number: str) -> phonenumbers.PhoneNumber | None: + """ + We often (up to ~3% of the time) see numbers which are not technically valid, but are close-enough-to-valid + that we want to give our users benefit of the doubt. + + We don't want to do this for every number, because if someone passes in a valid international number that + like "+1 (500) 555-1234" then we don't want to remove the + sign as we may then confuse it with a UK landline + + This includes numbers like: + + "0+447700900100" (a stray leading 0) + "000007700900100" (five leading zeros) + "+07700900100" (a leading plus but no country code) + "0+44(0)7700900100" (a mix of all of the above) + """ + with suppress(phonenumbers.NumberParseException): + # phonenumbers assumes a number without a + or 00 at beginning is always a local number. Given that we + # know excel sometimes strips these, if it doesn't parse as a UK number, lets try forcing it to be + # recognised as an international number + normalised_phone_number = phone_number.replace("+", "").lstrip("0") + + number = phonenumbers.parse(normalised_phone_number, "GB") + + if phonenumbers.is_possible_number(number): + return number + + return None + @staticmethod def _validate_forced_international_number(phone_number: str) -> phonenumbers.PhoneNumber | None: """ diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index de1e2dd76..9880699c8 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -409,7 +409,14 @@ def test_rejects_invalid_uk_landlines(self, phone_number): assert e.value.code == InvalidPhoneError.Codes.INVALID_NUMBER @pytest.mark.parametrize( - "phone_number, error_message", invalid_uk_mobile_phone_numbers + invalid_international_numbers + "phone_number, error_message", + invalid_uk_mobile_phone_numbers + + invalid_international_numbers + + [ + # french number - but 87 isn't a valid start of a phone number in france as defined by libphonenumber + # eg https://github.com/google/libphonenumber/blob/master/resources/metadata/33/ranges.csv + ("0033877123456", InvalidPhoneError.ERROR_MESSAGES[InvalidPhoneError.Codes.INVALID_NUMBER]), + ], ) def test_rejects_invalid_international_phone_numbers(self, phone_number, error_message): with pytest.raises(InvalidPhoneError): From 680dbbf0f4668c9d1ec1fe5bc2a68941fa1b497d Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 3 Jul 2024 15:53:45 +0100 Subject: [PATCH 085/211] add some more test cases for validation From a million phone numbers validated succesfully with our old code, approximately 3.5% of these do not validate with the new phonenumbers based code. All these test cases were derived from those 3.5% - we then took those phone numbers, anonymised them (replacing the last seven digits) while keeping all syntax/whitespace/area codes the same, and then removing test cases that looked the same to get a list. In time we may want to remove these tests or integrate them with the rest of the test suite. For succesful ones, mark what we think they should be normalised to, and mark the error codes for the others. --- .../recipient_validation/test_phone_number.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index 9880699c8..e4de923bd 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -485,3 +485,61 @@ def test_get_normalised_format_works_for_international_numbers(self, phone_numbe ) def test_get_human_readable_format(self, phone_number, expected_formatted): assert PhoneNumber(phone_number, allow_international=True).get_human_readable_format() == expected_formatted + + # TODO: when we've removed the old style validation, we can just roll these in to our regular test fixtures + # eg valid_uk_landline, invalid_uk_mobile_number, valid_international_number + @pytest.mark.parametrize( + "phone_number, expected_normalised_number", + [ + # probably UK numbers + ("+07044123456", "447044123456"), + ("0+44(0)7779123456", "447779123456"), + ("0+447988123456", "447988123456"), + ("00447911123456", "447911123456"), + ("04407379123456", "447379123456"), + ("0447300123456", "447300123456"), + ("000007392123456", "447392123456"), + ("0007465123456", "447465123456"), + ("007341123456", "447341123456"), + # could be a UK landline, could be a US number. We assume UK landlines + ("001708123456", "441708123456"), + ("+01158123456", "441158123456"), + ("+01323123456", "441323123456"), + ("+03332123456", "443332123456"), + # probably german + ("+04915161123456", "4915161123456"), + ], + ) + def test_validate_normalised_succeeds(self, phone_number, expected_normalised_number): + normalised_number = PhoneNumber(phone_number, allow_international=True) + assert str(normalised_number) == expected_normalised_number + + # TODO: decide if all these tests are useful to have. + # they represent real (but obfuscated/anonymised) phone numbers that notify has sent to recently that + # validated with the old code, but not with the new phonenumbers code + @pytest.mark.parametrize( + "phone_number, expected_error_code", + [ + ("(07417)4123456", InvalidPhoneError.Codes.TOO_LONG), + ("(06)25123456", InvalidPhoneError.Codes.INVALID_NUMBER), + ("+00263 71123456", InvalidPhoneError.Codes.INVALID_NUMBER), + ("+0065951123456", InvalidPhoneError.Codes.TOO_LONG), + ("00129123456", InvalidPhoneError.Codes.INVALID_NUMBER), + ("003570123456", InvalidPhoneError.Codes.INVALID_NUMBER), + ("0038097123456", InvalidPhoneError.Codes.TOO_LONG), + ("00407833123456", InvalidPhoneError.Codes.TOO_LONG), + ("0041903123456", InvalidPhoneError.Codes.INVALID_NUMBER), + ("005915209123456", InvalidPhoneError.Codes.TOO_LONG), + ("00617584123456", InvalidPhoneError.Codes.INVALID_NUMBER), + ("0064 495123456", InvalidPhoneError.Codes.INVALID_NUMBER), + ("00667123456", InvalidPhoneError.Codes.INVALID_NUMBER), + ("0092363123456", InvalidPhoneError.Codes.INVALID_NUMBER), + ("009677337123456", InvalidPhoneError.Codes.TOO_LONG), + ("047354123456", InvalidPhoneError.Codes.TOO_LONG), + ("0049 160 123456", InvalidPhoneError.Codes.INVALID_NUMBER), + ], + ) + def test_validate_normalised_fails(self, phone_number, expected_error_code): + with pytest.raises(InvalidPhoneError) as exc: + PhoneNumber(phone_number, allow_international=True) + assert exc.value.code == expected_error_code From c4971ac9d395054cc9c6625f4effd3cd4b5aa793 Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Wed, 3 Jul 2024 16:57:43 +0100 Subject: [PATCH 086/211] normalise numbers and retry if they fail validation some phone numbers fail validation because they're a bit weird. Perhaps theres been some overenthusiastic slightly incorrect normalisation carried out by the users. Try and normalise the phone number - remove any leading 0s and +s - and then try again --- .../recipient_validation/phone_number.py | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 3fd814979..06884877b 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -165,7 +165,11 @@ class PhoneNumber: def __init__(self, phone_number: str, *, allow_international: bool) -> None: self.raw_input = phone_number self.allow_international = allow_international - self.number = self.validate_phone_number(phone_number) + try: + self.number = self.validate_phone_number(phone_number) + except InvalidPhoneError: + phone_number = self._aggressively_normalise_number(phone_number) + self.number = self.validate_phone_number(phone_number) @staticmethod def _raise_if_phone_number_contains_invalid_characters(number: str) -> None: @@ -202,9 +206,7 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE) if (reason := phonenumbers.is_possible_number_with_reason(number)) != phonenumbers.ValidationResult.IS_POSSIBLE: - if normalised_number := self._validate_aggressively_normalised_number(phone_number): - number = normalised_number - elif self.allow_international and ( + if self.allow_international and ( forced_international_number := self._validate_forced_international_number(phone_number) ): number = forced_international_number @@ -220,7 +222,7 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: return number @staticmethod - def _validate_aggressively_normalised_number(phone_number: str) -> phonenumbers.PhoneNumber | None: + def _aggressively_normalise_number(phone_number: str) -> str: """ We often (up to ~3% of the time) see numbers which are not technically valid, but are close-enough-to-valid that we want to give our users benefit of the doubt. @@ -235,18 +237,7 @@ def _validate_aggressively_normalised_number(phone_number: str) -> phonenumbers. "+07700900100" (a leading plus but no country code) "0+44(0)7700900100" (a mix of all of the above) """ - with suppress(phonenumbers.NumberParseException): - # phonenumbers assumes a number without a + or 00 at beginning is always a local number. Given that we - # know excel sometimes strips these, if it doesn't parse as a UK number, lets try forcing it to be - # recognised as an international number - normalised_phone_number = phone_number.replace("+", "").lstrip("0") - - number = phonenumbers.parse(normalised_phone_number, "GB") - - if phonenumbers.is_possible_number(number): - return number - - return None + return phone_number.replace("+", "").lstrip("0") @staticmethod def _validate_forced_international_number(phone_number: str) -> phonenumbers.PhoneNumber | None: From c761829c5a8c14fe72f065809fd9578d369bbc52 Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Tue, 9 Jul 2024 11:54:30 +0100 Subject: [PATCH 087/211] Changes following PR review: - Fix typo - Move TODO marker from validation message into comment - Add underscore for full snake case in function name - Rename private function --- notifications_utils/recipient_validation/errors.py | 4 ++-- notifications_utils/recipient_validation/phone_number.py | 6 +++--- tests/recipient_validation/test_phone_number.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/notifications_utils/recipient_validation/errors.py b/notifications_utils/recipient_validation/errors.py index 0dd7d9495..ead01882b 100644 --- a/notifications_utils/recipient_validation/errors.py +++ b/notifications_utils/recipient_validation/errors.py @@ -28,7 +28,7 @@ class Codes(StrEnum): # this catches numbers with the right length but wrong digits # for example UK numbers cannot start "06" as that hasn't been assigned to a purpose by ofcom, # or a 9 digit UK number that does not start "01" or "0800". - Codes.INVALID_NUMBER: "TODO: CONTENT! Number is not valid – double check the mobile number you entered", + Codes.INVALID_NUMBER: "Number is not valid – double check the phone number you entered", # TODO: CONTENT! Codes.TOO_LONG: "Mobile number is too long", Codes.TOO_SHORT: "Mobile number is too short", Codes.NOT_A_UK_MOBILE: ( @@ -55,7 +55,7 @@ def __init__(self, *, code: Codes = Codes.INVALID_NUMBER): super().__init__(message=self.ERROR_MESSAGES[code]) @classmethod - def from_phonenumbers_validationresult(cls, reason: phonenumbers.ValidationResult) -> str: + def from_phonenumbers_validation_result(cls, reason: phonenumbers.ValidationResult) -> str: match reason: case phonenumbers.ValidationResult.TOO_LONG: code = cls.Codes.TOO_LONG diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 06884877b..54f39c623 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -168,7 +168,7 @@ def __init__(self, phone_number: str, *, allow_international: bool) -> None: try: self.number = self.validate_phone_number(phone_number) except InvalidPhoneError: - phone_number = self._aggressively_normalise_number(phone_number) + phone_number = self._thoroughly_normalise_number(phone_number) self.number = self.validate_phone_number(phone_number) @staticmethod @@ -211,7 +211,7 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: ): number = forced_international_number else: - raise InvalidPhoneError.from_phonenumbers_validationresult(reason) + raise InvalidPhoneError.from_phonenumbers_validation_result(reason) if not phonenumbers.is_valid_number(number): # is_possible just checks the length of a number for that country/region. is_valid checks if it's @@ -222,7 +222,7 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: return number @staticmethod - def _aggressively_normalise_number(phone_number: str) -> str: + def _thoroughly_normalise_number(phone_number: str) -> str: """ We often (up to ~3% of the time) see numbers which are not technically valid, but are close-enough-to-valid that we want to give our users benefit of the doubt. diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index e4de923bd..cdef7ac48 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -393,7 +393,7 @@ def test_format_phone_number_human_readable_doenst_throw(): assert format_phone_number_human_readable("ALPHANUM3R1C") == "ALPHANUM3R1C" -class TestPhoneNumbeClass: +class TestPhoneNumberClass: @pytest.mark.parametrize("phone_number, error_message", invalid_uk_mobile_phone_numbers) def test_rejects_invalid_uk_mobile_phone_numbers(self, phone_number, error_message): From e66063570c88ce10b3a9e2c3971236240251a0dc Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Tue, 9 Jul 2024 12:24:54 +0100 Subject: [PATCH 088/211] Version bump --- CHANGELOG.md | 6 ++++++ notifications_utils/version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d5ac554b..1d17da14d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # CHANGELOG +## 81.1.0 +* introduce new validation class - `PhoneNumber`, that we will use for services that want to send sms +to landline (and in the future this new code can be extended for all phone number validation) +* in this new class, we use `phonenumbers` library for validating phone numbers, instead of our custom valdiation code + + ## 81.0.0 * BREAKING CHANGE: The constructor for `notification_utils.recipient_validation.errors.InvalidPhoneError` diff --git a/notifications_utils/version.py b/notifications_utils/version.py index ba881669a..85c3a00ed 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "81.0.0" # 151ab871231fba7feb2eb8c23c908da9 +__version__ = "81.1.0" # 151ab871231fba7feb2eb8c23c908da9 From ef7a74cc429f7c76ec106482cf1bd4143620e120 Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Mon, 15 Jul 2024 14:14:15 +0100 Subject: [PATCH 089/211] The phonenumbers library (correctly) does not consider telephone numbers that OFCOM has reserved for use in TV and radio drama programs https://www.ofcom.org.uk/phones-and-broadband/phone-numbers/numbers-for-drama/ several tests in notifications-api, and a small percentage (~0.05%) of notifications sent on notify currently use these numbers. This fix adds logic allowing TV numbers to UK mobiles to pass validation. --- .../recipient_validation/phone_number.py | 17 ++++++++++++++++- notifications_utils/version.py | 2 +- tests/recipient_validation/test_phone_number.py | 8 ++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 54f39c623..051b27e98 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -1,3 +1,4 @@ +import re from collections import namedtuple from contextlib import suppress @@ -217,10 +218,24 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: # is_possible just checks the length of a number for that country/region. is_valid checks if it's # a valid sequence of numbers. This doesn't cover "is this number registered to an MNO". # For example UK numbers cannot start "06" as that hasn't been assigned to a purpose by ofcom - raise InvalidPhoneError(code=InvalidPhoneError.Codes.INVALID_NUMBER) + if self._is_tv_number(number): + return number + else: + raise InvalidPhoneError(code=InvalidPhoneError.Codes.INVALID_NUMBER) return number + @staticmethod + def _is_tv_number(phone_number) -> bool: + """ + The phonenumbers library does not consider TV numbers (fake numbers OFCOM reserves use in TV, film etc) + valid. This method checks whether a normalised phone number that has failed the library's validation is + in fact a valid TV number + """ + phone_number_as_string = str(phone_number.national_number) + if re.match("7700[900000-900999]", phone_number_as_string): + return True + @staticmethod def _thoroughly_normalise_number(phone_number: str) -> str: """ diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 85c3a00ed..8de39439d 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "81.1.0" # 151ab871231fba7feb2eb8c23c908da9 +__version__ = "81.1.1" # 43e5473167e33eed2d599e31aeda43ef diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index cdef7ac48..87b0a2d04 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -543,3 +543,11 @@ def test_validate_normalised_fails(self, phone_number, expected_error_code): with pytest.raises(InvalidPhoneError) as exc: PhoneNumber(phone_number, allow_international=True) assert exc.value.code == expected_error_code + + @pytest.mark.parametrize( + "phone_number, expected_valid_number", + [("07700900010", "447700900010"), ("447700900020", "447700900020"), ("+447700900030", "447700900030")], + ) + def test_tv_number_passes(self, phone_number, expected_valid_number): + number = PhoneNumber(phone_number, allow_international=True) + assert expected_valid_number == str(number) From 664d78c81c1265f60ffac8acf1ad4666df7f6f01 Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Thu, 18 Jul 2024 09:18:41 +0100 Subject: [PATCH 090/211] added missing changelog for 81.1.1 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d17da14d..3857df9fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # CHANGELOG +## 81.1.1 +* Adds condition to validation to allow TV Numbers (https://www.ofcom.org.uk/phones-and-broadband/phone-numbers/numbers-for-drama/) for UK mobiles + ## 81.1.0 * introduce new validation class - `PhoneNumber`, that we will use for services that want to send sms to landline (and in the future this new code can be extended for all phone number validation) From 568a6e56832499b532a763a273a3efa61bf78e45 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Wed, 17 Jul 2024 12:04:56 +0100 Subject: [PATCH 091/211] Add validation for fixed abode addresses to `PostalAddress` The `has_no_fixed_abode_address` method checks that no address lines simply consist of "NFA" or "NFA," and that "no fixed address" or "no fixed abode" is not in the address (all case insensitive). Letters sent to no fixed abode addresses have been causing issues since they often contain invalid addresses and weren't intended to be sent. --- .../recipient_validation/postal_address.py | 14 ++++ .../test_postal_address.py | 84 +++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/notifications_utils/recipient_validation/postal_address.py b/notifications_utils/recipient_validation/postal_address.py index 59c990228..08d0acaaa 100644 --- a/notifications_utils/recipient_validation/postal_address.py +++ b/notifications_utils/recipient_validation/postal_address.py @@ -146,6 +146,19 @@ def has_invalid_characters(self): line.startswith(tuple(self.INVALID_CHARACTERS_AT_START_OF_ADDRESS_LINE)) for line in self.normalised_lines ) + @property + def has_no_fixed_abode_address(self): + """ + We don't want users to sent to no fixed abode addresses, so validate that + - no lines just consist of "NFA" (case insensitive) + - the address does not contain "no fixed abode" or "no fixed address" (case insensitive) + """ + if any(line.lower() == "nfa" for line in self.normalised_lines): + return True + if re.search(r"no fixed (abode|address)", self.normalised, re.IGNORECASE): + return True + return False + @property def has_invalid_country_for_bfpo_address(self): """We don't want users to specify the country if they provide a BFPO number. Some BFPO numbers may resolve @@ -220,6 +233,7 @@ def valid(self): and not self.has_too_many_lines and not self.has_invalid_characters and not (self.international and self.is_bfpo_address) + and not self.has_no_fixed_abode_address ) diff --git a/tests/recipient_validation/test_postal_address.py b/tests/recipient_validation/test_postal_address.py index e81145012..2a96877c2 100644 --- a/tests/recipient_validation/test_postal_address.py +++ b/tests/recipient_validation/test_postal_address.py @@ -275,6 +275,75 @@ def test_has_invalid_characters(address, expected_result): assert PostalAddress(address).has_invalid_characters is expected_result +@pytest.mark.parametrize( + "address, expected_result", + [ + ( + "", + False, + ), + ( + """ + 123 Example Street + NFA NFA2024 + SW1 A 1 AA + """, + False, + ), + ( + """ + User with no Fixed Address, + London + SW1 A 1 AA + """, + True, + ), + ( + """ + A Person + NFA + SW1A 1AA + """, + True, + ), + ( + """ + A Person + NFA, + SW1A 1AA + """, + True, + ), + ( + """ + A Person + no fixed Abode + SW1A 1AA + """, + True, + ), + ( + """ + A Person + NO FIXED ADDRESS + SW1A 1AA + """, + True, + ), + ( + """ + nfa + Berlin + Deutschland + """, + True, + ), + ], +) +def test_has_no_fixed_abode_address(address, expected_result): + assert PostalAddress(address).has_no_fixed_abode_address is expected_result + + @pytest.mark.parametrize( "address, expected_international", ( @@ -747,6 +816,15 @@ def test_format_postcode_for_printing(postcode, postcode_with_space): True, False, ), + ( + """ + House + No fixed abode + France + """, + False, + False, + ), ( """ No postcode or country @@ -841,6 +919,12 @@ def test_valid_with_invalid_characters(): assert PostalAddress(address, allow_international_letters=True).valid is False +def test_valid_with_nfa_address(): + postal_address = PostalAddress("User\nNo fixed abode\nSW1 1AA") + assert postal_address.valid is False + assert postal_address.has_valid_last_line is True + + @pytest.mark.parametrize( "international, expected_valid", ( From 6585998e48726b5352e14648c3dc9bf7b750dfaf Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Wed, 17 Jul 2024 15:42:14 +0100 Subject: [PATCH 092/211] Bump version from 81.1.0 to 82.0.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3857df9fb..f518d6623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 82.0.0 + +* Change `PostalAddress` to add `has_no_fixed_abode_address` method. No fixed abode addresses are now considered invalid. + ## 81.1.1 * Adds condition to validation to allow TV Numbers (https://www.ofcom.org.uk/phones-and-broadband/phone-numbers/numbers-for-drama/) for UK mobiles diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 8de39439d..064778e1b 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "81.1.1" # 43e5473167e33eed2d599e31aeda43ef +__version__ = "82.0.0" # bcab8038b94ee470f6dae8bd7893619f From d2327b6fad8ec7fec6d85acf02bb8a2065ed6b05 Mon Sep 17 00:00:00 2001 From: Ben Corlett Date: Mon, 22 Jul 2024 09:36:26 +0100 Subject: [PATCH 093/211] Add new web request log fields. Namely, environment_name, request_size and response_size --- notifications_utils/logging/__init__.py | 3 + tests/test_logging.py | 141 ++++++++++++++++++++++++ 2 files changed, 144 insertions(+) diff --git a/notifications_utils/logging/__init__.py b/notifications_utils/logging/__init__.py index a251265ee..fc9b78c7e 100644 --- a/notifications_utils/logging/__init__.py +++ b/notifications_utils/logging/__init__.py @@ -25,6 +25,8 @@ def _common_request_extra_log_context(): return { "method": request.method, "url": request.url, + "environment": current_app.config["NOTIFY_ENVIRONMENT"] if "NOTIFY_ENVIRONMENT" in current_app.config else "", + "request_size": len(request.data), "endpoint": request.endpoint, "remote_addr": request.remote_addr, "user_agent": request.user_agent.string, @@ -81,6 +83,7 @@ def after_request(response): if hasattr(request, "before_request_real_time") else None ), + "response_size": response.calculate_content_length(), **_common_request_extra_log_context(), } current_app.logger.getChild("request").log( diff --git a/tests/test_logging.py b/tests/test_logging.py index 29caff0b5..b0857b5b7 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -99,6 +99,7 @@ def test_app_request_logs_level_by_status_code( with_request_helper, ): app = app_with_mocked_logger + app.config["NOTIFY_ENVIRONMENT"] = "foo" mock_req_logger = mock.Mock( spec=builtin_logging.Logger("flask.app.request"), handlers=[], @@ -126,6 +127,8 @@ def some_route(): "endpoint": "some_route", "host": "localhost", "path": "/", + "environment": "foo", + "request_size": 0, "user_agent": AnyStringMatching("Werkzeug.*"), "remote_addr": "127.0.0.1", "parent_span_id": "deadbeef" if with_request_helper else None, @@ -136,6 +139,8 @@ def some_route(): "method": "GET", "endpoint": "some_route", "host": "localhost", + "environment": "foo", + "request_size": 0, "path": "/", "user_agent": AnyStringMatching("Werkzeug.*"), "remote_addr": "127.0.0.1", @@ -156,6 +161,9 @@ def some_route(): "path": "/", "user_agent": AnyStringMatching("Werkzeug.*"), "method": "GET", + "environment": "foo", + "request_size": 0, + "response_size": 3, "endpoint": "some_route", "remote_addr": "127.0.0.1", "parent_span_id": "deadbeef" if with_request_helper else None, @@ -169,6 +177,9 @@ def some_route(): "path": "/", "user_agent": AnyStringMatching("Werkzeug.*"), "method": "GET", + "environment": "foo", + "request_size": 0, + "response_size": 3, "endpoint": "some_route", "remote_addr": "127.0.0.1", "parent_span_id": "deadbeef" if with_request_helper else None, @@ -207,6 +218,8 @@ def some_route(): "method": "GET", "endpoint": "some_route", "host": "localhost", + "environment": "", + "request_size": 0, "path": "/", "user_agent": AnyStringMatching("Werkzeug.*"), "remote_addr": "127.0.0.1", @@ -218,6 +231,8 @@ def some_route(): "method": "GET", "endpoint": "some_route", "host": "localhost", + "environment": "", + "request_size": 0, "path": "/", "user_agent": AnyStringMatching("Werkzeug.*"), "remote_addr": "127.0.0.1", @@ -236,6 +251,9 @@ def some_route(): "url": "http://localhost/", "method": "GET", "endpoint": "some_route", + "environment": "", + "request_size": 0, + "response_size": RestrictedAny(lambda x: x > 10), "host": "localhost", "path": "/", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -249,6 +267,9 @@ def some_route(): "url": "http://localhost/", "method": "GET", "endpoint": "some_route", + "environment": "", + "request_size": 0, + "response_size": RestrictedAny(lambda x: x > 10), "host": "localhost", "path": "/", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -265,6 +286,7 @@ def some_route(): def test_app_request_logs_response_on_status_200(app_with_mocked_logger): app = app_with_mocked_logger + app.config["NOTIFY_ENVIRONMENT"] = "bar" mock_req_logger = mock.Mock( spec=builtin_logging.Logger("flask.app.request"), handlers=[], @@ -295,6 +317,9 @@ def metrics(): "url": "http://localhost/_status", "method": "GET", "endpoint": "status", + "environment": "bar", + "request_size": 0, + "response_size": 2, "host": "localhost", "path": "/_status", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -308,6 +333,9 @@ def metrics(): "url": "http://localhost/_status", "method": "GET", "endpoint": "status", + "environment": "bar", + "request_size": 0, + "response_size": 2, "host": "localhost", "path": "/_status", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -331,6 +359,9 @@ def metrics(): "url": "http://localhost/metrics", "method": "GET", "endpoint": "metrics", + "environment": "bar", + "request_size": 0, + "response_size": 2, "host": "localhost", "path": "/metrics", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -344,6 +375,9 @@ def metrics(): "url": "http://localhost/metrics", "method": "GET", "endpoint": "metrics", + "environment": "bar", + "request_size": 0, + "response_size": 2, "host": "localhost", "path": "/metrics", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -368,6 +402,9 @@ def metrics(): "url": "http://localhost/_status", "method": "GET", "endpoint": "status", + "environment": "bar", + "request_size": 0, + "response_size": 4, "host": "localhost", "path": "/_status", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -381,6 +418,9 @@ def metrics(): "url": "http://localhost/_status", "method": "GET", "endpoint": "status", + "environment": "bar", + "request_size": 0, + "response_size": 4, "host": "localhost", "path": "/_status", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -415,6 +455,8 @@ def test_app_request_logs_responses_on_unknown_route(app_with_mocked_logger): "url": "http://localhost/foo", "method": "GET", "endpoint": None, + "environment": "", + "request_size": 0, "host": "localhost", "path": "/foo", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -426,6 +468,8 @@ def test_app_request_logs_responses_on_unknown_route(app_with_mocked_logger): "url": "http://localhost/foo", "method": "GET", "endpoint": None, + "environment": "", + "request_size": 0, "host": "localhost", "path": "/foo", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -445,6 +489,9 @@ def test_app_request_logs_responses_on_unknown_route(app_with_mocked_logger): "url": "http://localhost/foo", "method": "GET", "endpoint": None, + "environment": "", + "request_size": 0, + "response_size": RestrictedAny(lambda x: x > 0), "host": "localhost", "path": "/foo", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -458,6 +505,9 @@ def test_app_request_logs_responses_on_unknown_route(app_with_mocked_logger): "url": "http://localhost/foo", "method": "GET", "endpoint": None, + "environment": "", + "request_size": 0, + "response_size": RestrictedAny(lambda x: x > 0), "host": "localhost", "path": "/foo", "user_agent": AnyStringMatching("Werkzeug.*"), @@ -472,6 +522,97 @@ def test_app_request_logs_responses_on_unknown_route(app_with_mocked_logger): ) +def test_app_request_logs_responses_on_post(app_with_mocked_logger): + app = app_with_mocked_logger + mock_req_logger = mock.Mock( + spec=builtin_logging.Logger("flask.app.request"), + handlers=[], + ) + app.logger.getChild.side_effect = lambda name: mock_req_logger if name == "request" else mock.DEFAULT + + @app.route("/post", methods=["POST"]) + def post(): + return "OK", 200 + + logging.init_app(app) + + app.test_client().post("/post", data="foo=bar") + + assert ( + mock.call( + builtin_logging.DEBUG, + "Received request %(method)s %(url)s", + { + "url": "http://localhost/post", + "method": "POST", + "endpoint": "post", + "environment": "", + "request_size": 7, + "host": "localhost", + "path": "/post", + "user_agent": AnyStringMatching("Werkzeug.*"), + "remote_addr": "127.0.0.1", + "parent_span_id": None, + "process_": RestrictedAny(lambda value: isinstance(value, int)), + }, + extra={ + "url": "http://localhost/post", + "method": "POST", + "endpoint": "post", + "environment": "", + "request_size": 7, + "host": "localhost", + "path": "/post", + "user_agent": AnyStringMatching("Werkzeug.*"), + "remote_addr": "127.0.0.1", + "parent_span_id": None, + "process_": RestrictedAny(lambda value: isinstance(value, int)), + }, + ) + in mock_req_logger.log.call_args_list + ) + + assert ( + mock.call( + builtin_logging.INFO, + "%(method)s %(url)s %(status)s took %(request_time)ss", + { + "url": "http://localhost/post", + "method": "POST", + "endpoint": "post", + "environment": "", + "request_size": 7, + "response_size": 2, + "host": "localhost", + "path": "/post", + "user_agent": AnyStringMatching("Werkzeug.*"), + "remote_addr": "127.0.0.1", + "parent_span_id": None, + "status": 200, + "request_time": RestrictedAny(lambda value: isinstance(value, float)), + "process_": RestrictedAny(lambda value: isinstance(value, int)), + }, + extra={ + "url": "http://localhost/post", + "method": "POST", + "endpoint": "post", + "environment": "", + "request_size": 7, + "response_size": 2, + "host": "localhost", + "path": "/post", + "user_agent": AnyStringMatching("Werkzeug.*"), + "remote_addr": "127.0.0.1", + "parent_span_id": None, + "status": 200, + "request_time": RestrictedAny(lambda value: isinstance(value, float)), + "process_": RestrictedAny(lambda value: isinstance(value, int)), + }, + ) + in mock_req_logger.log.call_args_list + ) + + @pytest.mark.parametrize( "level_name,expected_level", ( From 7e0b5cd3c04b34378fc0cdf4019a43ab8eeb78fa Mon Sep 17 00:00:00 2001 From: Ben Corlett Date: Mon, 22 Jul 2024 09:39:26 +0100 Subject: [PATCH 094/211] Update changelog and version --- CHANGELOG.md | 3 +++ notifications_utils/version.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f518d6623..1633fee4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # CHANGELOG +## 82.1.0 +* Adds new logging fields to request logging. Namely environment_name, request_size and response_size + ## 82.0.0 * Change `PostalAddress` to add `has_no_fixed_abode_address` method. No fixed abode addresses are now considered invalid. diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 064778e1b..8dd64b39f 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.0.0" # bcab8038b94ee470f6dae8bd7893619f +__version__ = "82.1.0" # 450091ac5e62d2eb58f0bb3f9b5032a0 From 8e31cdd6306d73afe578102c1e38d1f4e3298d1e Mon Sep 17 00:00:00 2001 From: Ben Corlett Date: Thu, 25 Jul 2024 09:25:41 +0100 Subject: [PATCH 095/211] Fix the way we log the request_size. Accessing the data at this point can trigger a validation error early and cause a 500 error. --- CHANGELOG.md | 3 + notifications_utils/logging/__init__.py | 2 +- notifications_utils/version.py | 2 +- tests/test_logging.py | 99 +++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1633fee4c..1212df9cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # CHANGELOG +## 82.1.1 +* Fix the way we log the request_size. Accessing the data at this point can trigger a validation error early and cause a 500 error + ## 82.1.0 * Adds new logging fields to request logging. Namely environment_name, request_size and response_size diff --git a/notifications_utils/logging/__init__.py b/notifications_utils/logging/__init__.py index fc9b78c7e..0b5b811b9 100644 --- a/notifications_utils/logging/__init__.py +++ b/notifications_utils/logging/__init__.py @@ -26,7 +26,7 @@ def _common_request_extra_log_context(): "method": request.method, "url": request.url, "environment": current_app.config["NOTIFY_ENVIRONMENT"] if "NOTIFY_ENVIRONMENT" in current_app.config else "", - "request_size": len(request.data), + "request_size": request.content_length if request.content_length is not None else 0, "endpoint": request.endpoint, "remote_addr": request.remote_addr, "user_agent": request.user_agent.string, diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 8dd64b39f..55d5c0383 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.1.0" # 450091ac5e62d2eb58f0bb3f9b5032a0 +__version__ = "82.1.1" # 30e69c4868a9dd1981c0d79c8677d2c4 diff --git a/tests/test_logging.py b/tests/test_logging.py index b0857b5b7..e41a4311e 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -613,6 +613,105 @@ def post(): ) +def test_app_request_logs_responses_over_max_content(app_with_mocked_logger): + app = app_with_mocked_logger + + app.config["MAX_CONTENT_LENGTH"] = 3 * 1024 * 1024 + mock_req_logger = mock.Mock( + spec=builtin_logging.Logger("flask.app.request"), + handlers=[], + ) + app.logger.getChild.side_effect = lambda name: mock_req_logger if name == "request" else mock.DEFAULT + + @app.route("/post", methods=["POST"]) + def post(): + from flask import request + + # need to access data to trigger data too large error + _ = request.data + return "OK", 200 + + logging.init_app(app) + + file_content = b"a" * (3 * 1024 * 1024 + 1) + response = app.test_client().post("/post", data=file_content) + assert response.status_code == 413 + + assert ( + mock.call( + builtin_logging.DEBUG, + "Received request %(method)s %(url)s", + { + "url": "http://localhost/post", + "method": "POST", + "endpoint": "post", + "environment": "", + "request_size": (3 * 1024 * 1024 + 1), + "host": "localhost", + "path": "/post", + "user_agent": AnyStringMatching("Werkzeug.*"), + "remote_addr": "127.0.0.1", + "parent_span_id": None, + "process_": RestrictedAny(lambda value: isinstance(value, int)), + }, + extra={ + "url": "http://localhost/post", + "method": "POST", + "endpoint": "post", + "environment": "", + "request_size": 3145729, + "host": "localhost", + "path": "/post", + "user_agent": AnyStringMatching("Werkzeug.*"), + "remote_addr": "127.0.0.1", + "parent_span_id": None, + "process_": RestrictedAny(lambda value: isinstance(value, int)), + }, + ) + in mock_req_logger.log.call_args_list + ) + + assert ( + mock.call( + builtin_logging.INFO, + "%(method)s %(url)s %(status)s took %(request_time)ss", + { + "url": "http://localhost/post", + "method": "POST", + "endpoint": "post", + "environment": "", + "request_size": (3 * 1024 * 1024 + 1), + "response_size": RestrictedAny(lambda x: x > 0), + "host": "localhost", + "path": "/post", + "user_agent": AnyStringMatching("Werkzeug.*"), + "remote_addr": "127.0.0.1", + "parent_span_id": None, + "status": 413, + "request_time": RestrictedAny(lambda value: isinstance(value, float)), + "process_": RestrictedAny(lambda value: isinstance(value, int)), + }, + extra={ + "url": "http://localhost/post", + "method": "POST", + "endpoint": "post", + "environment": "", + "request_size": 3145729, + "response_size": RestrictedAny(lambda x: x > 0), + "host": "localhost", + "path": "/post", + "user_agent": AnyStringMatching("Werkzeug.*"), + "remote_addr": "127.0.0.1", + "parent_span_id": None, + "status": 413, + "request_time": RestrictedAny(lambda value: isinstance(value, float)), + "process_": RestrictedAny(lambda value: isinstance(value, int)), + }, + ) + in mock_req_logger.log.call_args_list + ) + + @pytest.mark.parametrize( "level_name,expected_level", ( From a48f05d5052a9c8130624f74757a812262971717 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 31 Jul 2024 10:54:09 +0100 Subject: [PATCH 096/211] Write to requirements.txt if no requirements.in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not all of our repos freeze their requirements from a `requirements.in` file into a `requirements.txt` file. Some have only a `requirements.txt` file which is edited directly. So our version script should write to that instead, if it can’t find a `requirements.in` file. --- CHANGELOG.md | 5 +++++ notifications_utils/version.py | 2 +- notifications_utils/version_tools.py | 9 +++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1212df9cc..03286f364 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG + +## 82.1.2 + +* Write updated version number to `requirements.txt` if no `requirements.in` file found + ## 82.1.1 * Fix the way we log the request_size. Accessing the data at this point can trigger a validation error early and cause a 500 error diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 55d5c0383..dc7bbd7bc 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.1.1" # 30e69c4868a9dd1981c0d79c8677d2c4 +__version__ = "82.1.2" # 277b159c4272d04e2cca81ea7a754fc9 diff --git a/notifications_utils/version_tools.py b/notifications_utils/version_tools.py index 501f249d6..42d0a80d8 100644 --- a/notifications_utils/version_tools.py +++ b/notifications_utils/version_tools.py @@ -58,11 +58,16 @@ def replace_line(line): return f"notifications-utils @ git+https://github.com/{repo_name}.git@{version}\n" return line + if requirements_file.exists(): + requirements_file_to_modify = requirements_file + else: + requirements_file_to_modify = frozen_requirements_file + new_requirements_file_contents = "".join( - replace_line(line) for line in requirements_file.read_text().splitlines(True) + replace_line(line) for line in requirements_file_to_modify.read_text().splitlines(True) ) - requirements_file.write_text(new_requirements_file_contents) + requirements_file_to_modify.write_text(new_requirements_file_contents) def get_relevant_changelog_lines(current_version, newest_version): From e151f66b730306badb5e7e51595fe1d0557e6676 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 26 Jul 2024 12:58:01 +0100 Subject: [PATCH 097/211] Add `unsubscribe_link` argument to email templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have started sending `list-unsubscribe` headers if a user marks a template as unsubscribeable, for teams who aren’t using the API. However we are already telling these teams to add unsubscribe links to the body of their emails. This means: - there are two different ways to add unsubscribe links - user will have to manage requests to unsubscribe coming from two different places This will be confusing, and make it harder for us to write good guidance that users will be able to follow, and harder to explain what the ‘let users unsubscribe from these emails’ checkbox does. If we have a confusing product proposition and users cant follow our guidance then we risk our spam score rising. This commit adds a way to insert an unsubscribe link (which we will generate) to the footer of every email. The API can then start generating these links, and the admin app can start responding to them. We can then report unsubscribes back to the team in the same way, no matter if the recipient has clicked the button in their email client or the link at the end of the email. --- CHANGELOG.md | 3 ++ notifications_utils/template.py | 20 +++++++++--- notifications_utils/version.py | 2 +- tests/test_template_types.py | 58 +++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03286f364..dbb7b509d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # CHANGELOG +## 82.2.0 + +* Add `unsubscribe_link` argument to email templates ## 82.1.2 diff --git a/notifications_utils/template.py b/notifications_utils/template.py index 01e60b2ea..a2de68b48 100644 --- a/notifications_utils/template.py +++ b/notifications_utils/template.py @@ -402,12 +402,22 @@ def placeholders(self): class BaseEmailTemplate(SubjectMixin, Template): template_type = "email" + def __init__(self, template, values=None, unsubscribe_link=None, **kwargs): + self.unsubscribe_link = unsubscribe_link + super().__init__(template, values, **kwargs) + + @property + def content_with_unsubscribe_link(self): + if self.unsubscribe_link: + return f"{self.content}\n\n---\n\n[Unsubscribe from these emails]({self.unsubscribe_link})" + return self.content + @property def html_body(self): return ( Take( Field( - self.content, + self.content_with_unsubscribe_link, self.values, html="escape", markdown_lists=True, @@ -462,7 +472,7 @@ def is_message_too_long(self): class PlainTextEmailTemplate(BaseEmailTemplate): def __str__(self): return ( - Take(Field(self.content, self.values, html="passthrough", markdown_lists=True)) + Take(Field(self.content_with_unsubscribe_link, self.values, html="passthrough", markdown_lists=True)) .then(unlink_govuk_escaped) .then(strip_unsupported_characters) .then(add_trailing_newline) @@ -505,8 +515,9 @@ def __init__( brand_colour=None, brand_banner=False, brand_alt_text=None, + **kwargs, ): - super().__init__(template, values) + super().__init__(template, values, **kwargs) self.govuk_banner = govuk_banner self.complete_html = complete_html self.brand_logo = brand_logo @@ -562,8 +573,9 @@ def __init__( reply_to=None, show_recipient=True, redact_missing_personalisation=False, + **kwargs, ): - super().__init__(template, values, redact_missing_personalisation=redact_missing_personalisation) + super().__init__(template, values, redact_missing_personalisation=redact_missing_personalisation, **kwargs) self.from_name = from_name self.reply_to = reply_to self.show_recipient = show_recipient diff --git a/notifications_utils/version.py b/notifications_utils/version.py index dc7bbd7bc..7823b12b1 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.1.2" # 277b159c4272d04e2cca81ea7a754fc9 +__version__ = "82.2.0" # dec63cb8da7dab6a8ff9993e711d3c96 diff --git a/tests/test_template_types.py b/tests/test_template_types.py index 4a796d58a..e81c9702b 100644 --- a/tests/test_template_types.py +++ b/tests/test_template_types.py @@ -2439,3 +2439,61 @@ def test_rendered_letter_template_for_print_can_toggle_notify_tag_and_always_hid ) assert ("content: 'NOTIFY';" in str(template)) == should_have_notify_tag assert "#mdi,\n #barcode,\n #qrcode {\n display: none;\n }" in str(template).strip() + + +@pytest.mark.parametrize( + "template_class, expected_content", + ( + ( + EmailPreviewTemplate, + ( + '
    ' + '

    ' + '' + "Unsubscribe from these emails" + "" + "

    \n" + ), + ), + ( + HTMLEmailTemplate, + ( + '
    ' + '

    ' + '' + "Unsubscribe from these emails" + "" + "

    \n" + ), + ), + ( + PlainTextEmailTemplate, + ( + "=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=\n" + "\n" + "Unsubscribe from these emails: https://www.example.com\n" + ), + ), + ), +) +def test_unsubscribe_link_is_rendered( + template_class, + expected_content, +): + assert expected_content in ( + str( + template_class( + {"content": "Hello world", "subject": "subject", "template_type": "email"}, + {}, + unsubscribe_link="https://www.example.com", + ) + ) + ) + assert expected_content not in ( + str( + template_class( + {"content": "Hello world", "subject": "subject", "template_type": "email"}, + {}, + ) + ) + ) From 260e7bd4697846af58887a7fb8a855251a94da8f Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Tue, 6 Aug 2024 08:45:16 +0100 Subject: [PATCH 098/211] Separate out check for if service can't send to international but tries to When this check was happening with all the rest of the number validation, when it failed we re-run it with a throoghly normalised number, which made phonenumbers libary think it's a UK landline that's too long for US numbers. When we do it separately, we raise the NOT_A_UK_MOBILE error as we should, and we don't try to overnormalise when we get it. --- CHANGELOG.md | 4 ++++ .../recipient_validation/phone_number.py | 23 ++++++++++++------- notifications_utils/version.py | 2 +- .../recipient_validation/test_phone_number.py | 15 ++++++++++++ 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbb7b509d..2ebd4d22e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 82.2.1 + +* Add fix to recipient_validation/phone_number.py to raise correct error if a service tries to send to an international number without that permission + ## 82.2.0 * Add `unsubscribe_link` argument to email templates diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 051b27e98..e0ea006b1 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -171,6 +171,20 @@ def __init__(self, phone_number: str, *, allow_international: bool) -> None: except InvalidPhoneError: phone_number = self._thoroughly_normalise_number(phone_number) self.number = self.validate_phone_number(phone_number) + self._raise_if_service_cannot_send_to_international_but_tries_to(phone_number) + + def _raise_if_service_cannot_send_to_international_but_tries_to(self, phone_number): + number = self._try_parse_number(phone_number) + if not self.allow_international and str(number.country_code) != UK_PREFIX: + raise InvalidPhoneError(code=InvalidPhoneError.Codes.NOT_A_UK_MOBILE) + + @staticmethod + def _try_parse_number(phone_number): + try: + # parse number as GB - if there's no country code, try and parse it as a UK number + return phonenumbers.parse(phone_number, "GB") + except phonenumbers.NumberParseException as e: + raise InvalidPhoneError(code=InvalidPhoneError.Codes.INVALID_NUMBER) from e @staticmethod def _raise_if_phone_number_contains_invalid_characters(number: str) -> None: @@ -194,14 +208,7 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: # of those cases separately before we parse with the phonenumbers library self._raise_if_phone_number_contains_invalid_characters(phone_number) - try: - # parse number as GB - if there's no country code, try and parse it as a UK number - number = phonenumbers.parse(phone_number, "GB") - except phonenumbers.NumberParseException as e: - raise InvalidPhoneError(code=InvalidPhoneError.Codes.INVALID_NUMBER) from e - - if not self.allow_international and str(number.country_code) != UK_PREFIX: - raise InvalidPhoneError(code=InvalidPhoneError.Codes.NOT_A_UK_MOBILE) + number = self._try_parse_number(phone_number) if str(number.country_code) not in COUNTRY_PREFIXES + ["+44"]: raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE) diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 7823b12b1..247eb435e 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.2.0" # dec63cb8da7dab6a8ff9993e711d3c96 +__version__ = "82.2.1" # dec63cb8da7dab6a8ff9993e711d3c96 diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index 87b0a2d04..2d314966f 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -551,3 +551,18 @@ def test_validate_normalised_fails(self, phone_number, expected_error_code): def test_tv_number_passes(self, phone_number, expected_valid_number): number = PhoneNumber(phone_number, allow_international=True) assert expected_valid_number == str(number) + + @pytest.mark.parametrize( + "phone_number, expected_error_code", + [ + ("+14158961600", InvalidPhoneError.Codes.NOT_A_UK_MOBILE), + ("+3225484211", InvalidPhoneError.Codes.NOT_A_UK_MOBILE), + ("+1 202-483-3000", InvalidPhoneError.Codes.NOT_A_UK_MOBILE), + ("+7 495 308-78-41", InvalidPhoneError.Codes.NOT_A_UK_MOBILE), + ("+74953087842", InvalidPhoneError.Codes.NOT_A_UK_MOBILE), + ], + ) + def test_international_does_not_normalise_to_uk_number(self, phone_number, expected_error_code): + with pytest.raises(InvalidPhoneError) as exc: + PhoneNumber(phone_number, allow_international=False) + assert exc.value.code == expected_error_code From 4a65c780c12a687429e2f7605f043df3f129b989 Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Thu, 25 Jul 2024 08:57:45 +0100 Subject: [PATCH 099/211] Premium rate landline numbers are a known vector of attack this adds to the validation logic for phonenumbers to disallow premium numbers --- CHANGELOG.md | 5 +++-- .../recipient_validation/phone_number.py | 22 +++++++++++++++++-- notifications_utils/version.py | 2 +- .../recipient_validation/test_phone_number.py | 7 ++---- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ebd4d22e..4ce1ae768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,16 @@ # CHANGELOG +## 82.3.0 +* Extends the validation logic for the `PhoneNumber` class to disallow premium rate numbers + ## 82.2.1 * Add fix to recipient_validation/phone_number.py to raise correct error if a service tries to send to an international number without that permission ## 82.2.0 - * Add `unsubscribe_link` argument to email templates ## 82.1.2 - * Write updated version number to `requirements.txt` if no `requirements.in` file found ## 82.1.1 diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index e0ea006b1..a9ceecefc 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -16,6 +16,17 @@ UK_PREFIX = "44" +ALLOW_LIST = { + phonenumbers.PhoneNumberType.FIXED_LINE, + phonenumbers.PhoneNumberType.MOBILE, + phonenumbers.PhoneNumberType.FIXED_LINE_OR_MOBILE, # ambiguous case where a number could be either landline/mobile + phonenumbers.PhoneNumberType.UAN, + phonenumbers.PhoneNumberType.PERSONAL_NUMBER, +} + +DENY_LIST = [ + phonenumbers.PhoneNumberType.PREMIUM_RATE, +] international_phone_info = namedtuple( "PhoneNumber", @@ -220,8 +231,7 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: number = forced_international_number else: raise InvalidPhoneError.from_phonenumbers_validation_result(reason) - - if not phonenumbers.is_valid_number(number): + if not (phonenumbers.is_valid_number(number) & self._is_allowed_phone_number_type(number)): # is_possible just checks the length of a number for that country/region. is_valid checks if it's # a valid sequence of numbers. This doesn't cover "is this number registered to an MNO". # For example UK numbers cannot start "06" as that hasn't been assigned to a purpose by ofcom @@ -232,6 +242,14 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: return number + @staticmethod + def _is_allowed_phone_number_type(phone_number: phonenumbers.PhoneNumber) -> bool: + if phonenumbers.number_type(phone_number) in DENY_LIST: + return False + if phonenumbers.number_type(phone_number) in ALLOW_LIST: + return True + return False + @staticmethod def _is_tv_number(phone_number) -> bool: """ diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 247eb435e..c31ad6f02 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.2.1" # dec63cb8da7dab6a8ff9993e711d3c96 +__version__ = "82.3.0" # c6ba6a75ce00d7c25a2c8fbfe9fb1dcb diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index 2d314966f..db5c7811b 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -58,11 +58,6 @@ "020 7946 0991", # london "030 1234 5678", # non-geographic "0550 123 4567", # corporate numbering and voip services - "0800 123 4567", # freephone - "0800 123 456", # shorter freephone - "0800 11 11", # shortest freephone - "0845 46 46", # short premium - "0900 123 4567", # premium ] invalid_uk_landlines = [ @@ -71,6 +66,8 @@ "0300 46 46", # short but not 01x or 08x "0800 11 12", # short but not 01x or 08x "0845 46 31", # short but not 01x or 08x + "0845 46 46", # short premium + "0900 123 4567", # premium ] invalid_uk_mobile_phone_numbers = sum( From dd0121d34d06a17d90aa587210c2bf78304b2b08 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Thu, 1 Aug 2024 11:41:20 +0100 Subject: [PATCH 100/211] Add new areas to international billing rates This adds 5 new countries / regions that are now supported to `international_billing_rates.yml` so that we can send to them. --- .../international_billing_rates.yml | 65 +++++++++++++++++++ tests/test_international_billing_rates.py | 2 +- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/notifications_utils/international_billing_rates.yml b/notifications_utils/international_billing_rates.yml index 1b25f1798..28cbe9d6e 100644 --- a/notifications_utils/international_billing_rates.yml +++ b/notifications_utils/international_billing_rates.yml @@ -1371,6 +1371,19 @@ billable_units: 2 names: - Comoros +'291': + attributes: + alpha: 'YES' + comment: null + dlr: 'YES' + generic_sender: null + numeric: 'YES' + sc: null + sender_and_registration_info: null + text_restrictions: null + billable_units: 4 + names: + - Eritrea '297': attributes: alpha: null @@ -2204,6 +2217,19 @@ billable_units: 3 names: - Palau +'681': + attributes: + alpha: 'YES' + comment: null + dlr: Carrier DLR + generic_sender: null + numeric: 'YES' + sc: null + sender_and_registration_info: null + text_restrictions: null + billable_units: 4 + names: + - Wallis and Futuna '682': attributes: alpha: null @@ -2217,6 +2243,19 @@ billable_units: 3 names: - Cook Islands +'683': + attributes: + alpha: 'YES' + comment: null + dlr: Carrier DLR + generic_sender: null + numeric: 'YES' + sc: null + sender_and_registration_info: null + text_restrictions: null + billable_units: 4 + names: + - Niue '685': attributes: alpha: null @@ -2230,6 +2269,19 @@ billable_units: 3 names: - Samoa +'686': + attributes: + alpha: 'YES' + comment: null + dlr: Carrier DLR + generic_sender: null + numeric: 'YES' + sc: null + sender_and_registration_info: null + text_restrictions: null + billable_units: 4 + names: + - Kiribati '687': attributes: alpha: 'YES' @@ -2256,6 +2308,19 @@ billable_units: 2 names: - French Polynesia +'690': + attributes: + alpha: 'YES' + comment: null + dlr: Carrier DLR + generic_sender: null + numeric: 'YES' + sc: null + sender_and_registration_info: null + text_restrictions: null + billable_units: 4 + names: + - Tokelau '691': attributes: alpha: null diff --git a/tests/test_international_billing_rates.py b/tests/test_international_billing_rates.py index e5df93fc8..c45becc61 100644 --- a/tests/test_international_billing_rates.py +++ b/tests/test_international_billing_rates.py @@ -30,7 +30,7 @@ def test_international_billing_rates_are_in_correct_format(country_prefix, value def test_country_codes(): - assert len(COUNTRY_PREFIXES) == 215 + assert len(COUNTRY_PREFIXES) == 220 @pytest.mark.parametrize( From 74a24bdb94fd2b842628c848d073a736ce6ce214 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Wed, 7 Aug 2024 15:24:04 +0100 Subject: [PATCH 101/211] Bump version to 82.4.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ce1ae768..a865b83e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 82.4.0 + +* Add support for sending SMS to more international numbers. Sending to Eritrea, Wallis and Futuna, Niue, Kiribati, and Tokelau is now supported. + ## 82.3.0 * Extends the validation logic for the `PhoneNumber` class to disallow premium rate numbers diff --git a/notifications_utils/version.py b/notifications_utils/version.py index c31ad6f02..ae097c85a 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.3.0" # c6ba6a75ce00d7c25a2c8fbfe9fb1dcb +__version__ = "82.4.0" # 4b2b1200cd9552377dbbae4704df096b From 719c7a5ca49ce7b6a35504106522726f1f7c1792 Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Wed, 14 Aug 2024 15:24:21 +0100 Subject: [PATCH 102/211] updates recipientcsv to use new validation code for services with sms_to_uk_landlines enabled CSV validation currently fails on admin if a csv file contains a valid UK landline number. This update will let us add the required logic to admin, to validate against the latest validation code which allows UK landlines for services with this feature enabled --- CHANGELOG.md | 4 ++++ notifications_utils/recipients.py | 8 +++++++- notifications_utils/version.py | 2 +- tests/test_recipient_csv.py | 33 +++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a865b83e2..b56509198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 82.5.0 + +* Add support for validation of UK landlines for services with sms_to_uk_landlines enabled using CSV flow + ## 82.4.0 * Add support for sending SMS to more international numbers. Sending to Eritrea, Wallis and Futuna, Niue, Kiribati, and Tokelau is now supported. diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index 9f71c9d28..fa13021d5 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -15,6 +15,7 @@ from notifications_utils.insensitive_dict import InsensitiveDict from notifications_utils.recipient_validation import email_address, phone_number from notifications_utils.recipient_validation.errors import InvalidEmailError, InvalidPhoneError, InvalidRecipientError +from notifications_utils.recipient_validation.phone_number import PhoneNumber from notifications_utils.recipient_validation.postal_address import ( address_line_7_key, address_lines_1_to_6_and_postcode_keys, @@ -46,6 +47,7 @@ def __init__( remaining_messages=sys.maxsize, allow_international_sms=False, allow_international_letters=False, + allow_sms_to_uk_landline=False, should_validate=True, ): self.file_data = strip_all_whitespace(file_data, extra_characters=",") @@ -55,6 +57,7 @@ def __init__( self.template = template self.allow_international_sms = allow_international_sms self.allow_international_letters = allow_international_letters + self.allow_sms_to_uk_landline = allow_sms_to_uk_landline self.remaining_messages = remaining_messages self.rows_as_list = None self.should_validate = should_validate @@ -317,7 +320,10 @@ def _get_error_for_field(self, key, value): # noqa: C901 if self.template_type == "email": email_address.validate_email_address(value) if self.template_type == "sms": - phone_number.validate_phone_number(value, international=self.allow_international_sms) + if self.allow_sms_to_uk_landline: + PhoneNumber(value, allow_international=self.allow_international_sms) + else: + phone_number.validate_phone_number(value, international=self.allow_international_sms) except InvalidRecipientError as error: return str(error) diff --git a/notifications_utils/version.py b/notifications_utils/version.py index ae097c85a..a624eb068 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.4.0" # 4b2b1200cd9552377dbbae4704df096b +__version__ = "82.5.0" # 6f61f59189d11bdb2c96fc37faafe2d1 diff --git a/tests/test_recipient_csv.py b/tests/test_recipient_csv.py index 6271090da..e4e4ca3dc 100644 --- a/tests/test_recipient_csv.py +++ b/tests/test_recipient_csv.py @@ -737,6 +737,39 @@ def test_international_recipients(file_contents, rows_with_bad_recipients): assert _index_rows(recipients.rows_with_bad_recipients) == rows_with_bad_recipients +@pytest.mark.parametrize( + "file_contents,rows_with_bad_recipients", + [ + ( + """ + phone number + 800000000000 + 1234 + +447900123 + """, + {0, 1, 2}, + ), + ( + """ + phone number + +441709510122 + 020 3002 4300 + 44117 925 1001 + + """, + set(), + ), + ], +) +def test_sms_to_uk_landlines(file_contents, rows_with_bad_recipients): + recipients = RecipientCSV( + file_contents, + template=_sample_template("sms"), + allow_sms_to_uk_landline=True, + ) + assert _index_rows(recipients.rows_with_bad_recipients) == rows_with_bad_recipients + + def test_errors_when_too_many_rows(): recipients = RecipientCSV( "email address\n" + ("a@b.com\n" * 101), From 73935467df01332ecc89c1e8456cf376ac359639 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Wed, 21 Aug 2024 13:40:36 +0100 Subject: [PATCH 103/211] add LazyLocalGetter and tests --- notifications_utils/local_vars.py | 34 ++++++++++++++++++++++++ tests/test_local_vars.py | 44 +++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 notifications_utils/local_vars.py create mode 100644 tests/test_local_vars.py diff --git a/notifications_utils/local_vars.py b/notifications_utils/local_vars.py new file mode 100644 index 000000000..d91f8fbbd --- /dev/null +++ b/notifications_utils/local_vars.py @@ -0,0 +1,34 @@ +from collections.abc import Callable +from contextvars import ContextVar +from typing import Generic, TypeVar + +T = TypeVar("T") + + +class LazyLocalGetter(Generic[T]): + """ + Wrapper for lazily-constructed context-local resources + """ + + context_var: ContextVar[T | None] + factory: Callable[[], T] + + def __init__(self, context_var: ContextVar[T | None], factory: Callable[[], T]): + """ + Given a reference to a `context_var`, the resulting instance will be a callable that + returns the current context's contents of that `context_var`, pre-populating it with + the results of a (zero-argument) call to `factory` if it is empty or None + """ + self.context_var = context_var + self.factory = factory + + def __call__(self) -> T: + r = self.context_var.get(None) + if r is None: + r = self.factory() + self.context_var.set(r) + + return r + + def clear(self) -> None: + self.context_var.set(None) diff --git a/tests/test_local_vars.py b/tests/test_local_vars.py new file mode 100644 index 000000000..9a5175bfc --- /dev/null +++ b/tests/test_local_vars.py @@ -0,0 +1,44 @@ +from contextvars import ContextVar +from itertools import count +from unittest import mock + +from notifications_utils.local_vars import LazyLocalGetter + + +def test_lazy_local_getter_reuses_first_constructed(request): + # we're not supposed to construct ContextVars inside functions because they can't + # really be garbage-collected, but otherwise it's difficult to ensure we're getting + # a "clean" ContextVar for this test + cv = ContextVar(request.node.name) # ensure name is unique across test session + + factory = mock.Mock( + spec=("__call__",), + side_effect=(getattr(mock.sentinel, f"some_object_{i}") for i in count()), + ) + + llg = LazyLocalGetter(cv, factory) + + assert llg() is mock.sentinel.some_object_0 + assert llg() is mock.sentinel.some_object_0 + + assert factory.call_args_list == [mock.call()] # despite two accesses + + +def test_lazy_local_getter_clear(request): + # ...same caveat about locally-declared ContextVar... + cv = ContextVar(request.node.name) # ensure name is unique across test session + + factory = mock.Mock( + spec=("__call__",), + side_effect=(getattr(mock.sentinel, f"some_object_{i}") for i in count()), + ) + + llg = LazyLocalGetter(cv, factory) + + assert llg() is mock.sentinel.some_object_0 + assert factory.call_args_list == [mock.call()] + factory.reset_mock() + + llg.clear() + assert llg() is mock.sentinel.some_object_1 + assert factory.call_args_list == [mock.call()] From 17617df4786db1ee47de4992d1029c90ff36abd6 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Wed, 21 Aug 2024 13:46:27 +0100 Subject: [PATCH 104/211] Bump version to 82.6.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b56509198..e7bdab5d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 82.6.0 + +* Add LazyLocalGetter class for lazily-initialized context-local resources + ## 82.5.0 * Add support for validation of UK landlines for services with sms_to_uk_landlines enabled using CSV flow diff --git a/notifications_utils/version.py b/notifications_utils/version.py index a624eb068..0d92795b8 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.5.0" # 6f61f59189d11bdb2c96fc37faafe2d1 +__version__ = "82.6.0" # 5c32a8390c476 From 622b013bd5a434da28ad261f739a86a972947d9e Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Thu, 29 Aug 2024 13:51:56 +0100 Subject: [PATCH 105/211] Adds validation check to PhoneNumber so that the error `TOO_SHORT` is raised if an empty string is passed into the object as a phonenumber Users of the v2 api expect, and often require, consistent error messages to be returned from the API as their code often does comparisons against errors as strings to handle notify errors. In the case of empty strings, an error corresponding to InvalidPhoneError.Codes.TOO_SHORT is expected --- CHANGELOG.md | 4 ++++ notifications_utils/recipient_validation/phone_number.py | 8 ++++++++ notifications_utils/version.py | 2 +- tests/recipient_validation/test_phone_number.py | 8 ++++++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7bdab5d7..bed07cf6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +# 82.6.1 + +* Adds validation check to `PhoneNumber` so that it returns the expected error message `TOO_SHORT` if an empty string is passed. This has caused issues with users of the v2 API getting inconsistent error messages + ## 82.6.0 * Add LazyLocalGetter class for lazily-initialized context-local resources diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index a9ceecefc..118d46b8d 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -203,6 +203,11 @@ def _raise_if_phone_number_contains_invalid_characters(number: str) -> None: if chars - {*ALL_WHITESPACE + "()-+" + "0123456789"}: raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNKNOWN_CHARACTER) + @staticmethod + def _raise_if_phone_number_is_empty(number: str) -> None: + if number == "": + raise InvalidPhoneError(code=InvalidPhoneError.Codes.TOO_SHORT) + def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: """ Validate a phone number and return the PhoneNumber object @@ -215,6 +220,9 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: changes whether it is parsed as international or not. * Convert error codes to match existing Notify error codes """ + + self._raise_if_phone_number_is_empty(phone_number) + # notify's old validation code is stricter than phonenumbers in not allowing letters etc, so need to catch some # of those cases separately before we parse with the phonenumbers library self._raise_if_phone_number_contains_invalid_characters(phone_number) diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 0d92795b8..3c8b9b526 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.6.0" # 5c32a8390c476 +__version__ = "82.6.1" # b4df1ae664036399e4b5661f55b2d3e5 diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index db5c7811b..4dc663274 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -563,3 +563,11 @@ def test_international_does_not_normalise_to_uk_number(self, phone_number, expec with pytest.raises(InvalidPhoneError) as exc: PhoneNumber(phone_number, allow_international=False) assert exc.value.code == expected_error_code + + +def test_empty_phone_number_is_rejected_with_correct_v2_error_message(): + phone_number = "" + error_message = InvalidPhoneError(code=InvalidPhoneError.Codes.TOO_SHORT) + with pytest.raises(InvalidPhoneError) as e: + PhoneNumber(phone_number=phone_number, allow_international=True) + assert str(error_message) == str(e.value) From 0ae0a55fac78fc74d63d692f2c0afd8b97d32f3d Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Mon, 2 Sep 2024 15:21:05 +0100 Subject: [PATCH 106/211] small commit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 29a287cd5..d6fcbb79b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # notifications-utils -Shared Python code for GOV.UK Notify applications. Standardises how to do logging, rendering message templates, parsing spreadsheets, talking to external services and more. +Shared Python code for GOV.UK Notify applications. Standardises how to do logging, rendering message templates, parsing spreadsheets, talking to external services and more. ## Setting up From 22dbe7964fce552f060199d6f479322451dc95df Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Mon, 2 Sep 2024 15:21:16 +0100 Subject: [PATCH 107/211] small commit revert --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d6fcbb79b..29a287cd5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # notifications-utils -Shared Python code for GOV.UK Notify applications. Standardises how to do logging, rendering message templates, parsing spreadsheets, talking to external services and more. +Shared Python code for GOV.UK Notify applications. Standardises how to do logging, rendering message templates, parsing spreadsheets, talking to external services and more. ## Setting up From 1563a7c00e560f2ced9dc5aad42faa30b2db4526 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 30 Aug 2024 16:22:28 +0100 Subject: [PATCH 108/211] LazyLocalGetter: add expected_type mechanism this is principally to allow LazyLocalGetter instances to have their inner type inspected without requiring population, but to make sure this is never *wrong* it also enforces that type at population-time. --- notifications_utils/local_vars.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/notifications_utils/local_vars.py b/notifications_utils/local_vars.py index d91f8fbbd..d0647c9b5 100644 --- a/notifications_utils/local_vars.py +++ b/notifications_utils/local_vars.py @@ -12,20 +12,38 @@ class LazyLocalGetter(Generic[T]): context_var: ContextVar[T | None] factory: Callable[[], T] - - def __init__(self, context_var: ContextVar[T | None], factory: Callable[[], T]): + expected_type: type[T] | None + + def __init__( + self, + context_var: ContextVar[T | None], + factory: Callable[[], T], + expected_type: type[T] | None = None, + ): """ Given a reference to a `context_var`, the resulting instance will be a callable that returns the current context's contents of that `context_var`, pre-populating it with - the results of a (zero-argument) call to `factory` if it is empty or None + the results of a (zero-argument) call to `factory` if it is empty or None. + + If `expected_type` is specified, the `factory` call's return value is checked to be + of that type, but in return the `.expected_type` attribute is accessible without + triggering population. """ self.context_var = context_var self.factory = factory + self.expected_type = expected_type def __call__(self) -> T: r = self.context_var.get(None) if r is None: r = self.factory() + + # exact type testing here, none of your issubclass flexibility + if self.expected_type is not None and type(r) is not self.expected_type: + raise TypeError( + f"factory returned value (of type {type(r)}) that is not of the expected type {self.expected_type}" + ) + self.context_var.set(r) return r From d4e961a04fef7b58f22ac9f6ad7bfddde09866b2 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Mon, 2 Sep 2024 13:36:42 +0100 Subject: [PATCH 109/211] Bump version to 82.7.0 --- CHANGELOG.md | 6 +++++- notifications_utils/version.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bed07cf6c..0df7b1847 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,16 @@ # CHANGELOG +## 82.7.0 + +* Add `expected_type` mechanism to `LazyLocalGetter` + # 82.6.1 * Adds validation check to `PhoneNumber` so that it returns the expected error message `TOO_SHORT` if an empty string is passed. This has caused issues with users of the v2 API getting inconsistent error messages ## 82.6.0 -* Add LazyLocalGetter class for lazily-initialized context-local resources +* Add `LazyLocalGetter` class for lazily-initialized context-local resources ## 82.5.0 diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 3c8b9b526..d00cbe4ec 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.6.1" # b4df1ae664036399e4b5661f55b2d3e5 +__version__ = "82.7.0" # b21ed1702bc409f764d6898e729430b7 From d72195a650ec4cd63a689a1d6c57cd0c599a4398 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 9 Aug 2024 11:00:53 +0100 Subject: [PATCH 110/211] Bump test requirements to the latest versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mainly doing this because the output of Pytest is much nicer in version 8. But I figure it can’t hurt to get bug fixes and improvements from the other dependencies too. --- .pre-commit-config.yaml | 4 ++-- requirements_for_test_common.txt | 20 ++++++++++---------- tests/test_logging.py | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be6f150c0..3426777b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,12 +7,12 @@ repos: - id: check-yaml - id: debug-statements - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.3.7' + rev: 'v0.6.4' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black - rev: 24.4.0 + rev: 24.8.0 hooks: - id: black name: black (python) diff --git a/requirements_for_test_common.txt b/requirements_for_test_common.txt index 4e69f1f33..88bd3c6f4 100644 --- a/requirements_for_test_common.txt +++ b/requirements_for_test_common.txt @@ -1,12 +1,12 @@ -beautifulsoup4==4.11.1 -pytest==7.2.0 -pytest-env==0.8.1 -pytest-mock==3.9.0 -pytest-xdist==3.0.2 -pytest-testmon==2.1.0 +beautifulsoup4==4.12.3 +pytest==8.3.2 +pytest-env==1.1.3 +pytest-mock==3.14.0 +pytest-xdist==3.6.1 +pytest-testmon==2.1.1 pytest-watch==4.2.0 -requests-mock==1.10.0 -freezegun==1.2.2 +requests-mock==1.12.1 +freezegun==1.5.1 -black==24.4.0 # Also update `.pre-commit-config.yaml` if this changes -ruff==0.3.7 # Also update `.pre-commit-config.yaml` if this changes +black==24.8.0 # Also update `.pre-commit-config.yaml` if this changes +ruff==0.6.4 # Also update `.pre-commit-config.yaml` if this changes diff --git a/tests/test_logging.py b/tests/test_logging.py index e41a4311e..7d220ddb5 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -20,8 +20,8 @@ class App: handlers = logging.get_handlers(app, extra_filters=[]) assert len(handlers) == 1 - assert type(handlers[0]) == builtin_logging.StreamHandler - assert type(handlers[0].formatter) == logging.Formatter + assert type(handlers[0]) is builtin_logging.StreamHandler + assert type(handlers[0].formatter) is logging.Formatter def test_get_handlers_sets_up_logging_appropriately_without_debug(): @@ -37,8 +37,8 @@ class App: handlers = logging.get_handlers(app, extra_filters=[]) assert len(handlers) == 1 - assert type(handlers[0]) == builtin_logging.StreamHandler - assert type(handlers[0].formatter) == logging.JSONFormatter + assert type(handlers[0]) is builtin_logging.StreamHandler + assert type(handlers[0].formatter) is logging.JSONFormatter @pytest.mark.parametrize( From a9224a8a03be1221ccf3a90613ca9484ef54983f Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Wed, 14 Aug 2024 15:29:46 +0100 Subject: [PATCH 111/211] Minor version bump --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bed07cf6c..19a9b6edd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 82.7.0 + +* Bump versions of common test dependencies (run `make bootstrap` to copy these into your app) + # 82.6.1 * Adds validation check to `PhoneNumber` so that it returns the expected error message `TOO_SHORT` if an empty string is passed. This has caused issues with users of the v2 API getting inconsistent error messages diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 3c8b9b526..746a57847 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.6.1" # b4df1ae664036399e4b5661f55b2d3e5 +__version__ = "82.7.0" # 0351e385960ad4dcd89342dcb4563721 From 12e9aa6b3b48e61a632209cb8245e4a01417e195 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 23 Aug 2024 15:32:44 +0100 Subject: [PATCH 112/211] AntivirusClient: use a persistent requests session also remove the init_app construction method as this pattern conflicts with on-demand client construction, needed for managing thread-local instances --- .../clients/antivirus/antivirus_client.py | 13 ++++++++----- tests/clients/antivirus/test_antivirus_client.py | 8 ++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/notifications_utils/clients/antivirus/antivirus_client.py b/notifications_utils/clients/antivirus/antivirus_client.py index f723db310..74a165af3 100644 --- a/notifications_utils/clients/antivirus/antivirus_client.py +++ b/notifications_utils/clients/antivirus/antivirus_client.py @@ -21,13 +21,16 @@ def from_exception(cls, e): class AntivirusClient: + """ + A client for the antivirus API + + This class is not thread-safe. + """ + def __init__(self, api_host=None, auth_token=None): self.api_host = api_host self.auth_token = auth_token - - def init_app(self, app): - self.api_host = app.config["ANTIVIRUS_API_HOST"] - self.auth_token = app.config["ANTIVIRUS_API_KEY"] + self.requests_session = requests.Session() def scan(self, document_stream): headers = {"Authorization": f"Bearer {self.auth_token}"} @@ -35,7 +38,7 @@ def scan(self, document_stream): headers.update(request.get_onwards_request_headers()) try: - response = requests.post( + response = self.requests_session.post( f"{self.api_host}/scan", headers=headers, files={"document": document_stream}, diff --git a/tests/clients/antivirus/test_antivirus_client.py b/tests/clients/antivirus/test_antivirus_client.py index 173ac0152..f5bab891a 100644 --- a/tests/clients/antivirus/test_antivirus_client.py +++ b/tests/clients/antivirus/test_antivirus_client.py @@ -12,10 +12,10 @@ @pytest.fixture(scope="function") def app_antivirus_client(app, mocker): - client = AntivirusClient() - app.config["ANTIVIRUS_API_HOST"] = "https://antivirus" - app.config["ANTIVIRUS_API_KEY"] = "test-antivirus-key" - client.init_app(app) + client = AntivirusClient( + api_host="https://antivirus", + auth_token="test-antivirus-key", + ) return app, client From 08227b2e988dbad60fdb27192f0d52e61341b225 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 23 Aug 2024 15:57:02 +0100 Subject: [PATCH 113/211] ZendeskClient: use a persistent requests session also remove the init_app construction method as this pattern conflicts with on-demand client construction, needed for managing thread-local instances --- .../clients/zendesk/zendesk_client.py | 20 +++++++++++-------- tests/clients/zendesk/test_zendesk_client.py | 12 +++-------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/notifications_utils/clients/zendesk/zendesk_client.py b/notifications_utils/clients/zendesk/zendesk_client.py index c3d3f0a5a..35b09354e 100644 --- a/notifications_utils/clients/zendesk/zendesk_client.py +++ b/notifications_utils/clients/zendesk/zendesk_client.py @@ -48,6 +48,12 @@ class NotifySupportTicketComment: class ZendeskClient: + """ + A client for the Zendesk API + + This class is not thread-safe. + """ + # the account used to authenticate with. If no requester is provided, the ticket will come from this account. NOTIFY_ZENDESK_EMAIL = "zd-api-notify@digital.cabinet-office.gov.uk" @@ -55,14 +61,12 @@ class ZendeskClient: ZENDESK_UPDATE_TICKET_URL = "https://govuk.zendesk.com/api/v2/tickets/{ticket_id}" ZENDESK_UPLOAD_FILE_URL = "https://govuk.zendesk.com/api/v2/uploads.json" - def __init__(self): - self.api_key = None - - def init_app(self, app, *args, **kwargs): - self.api_key = app.config.get("ZENDESK_API_KEY") + def __init__(self, api_key): + self.api_key = api_key + self.requests_session = requests.Session() def send_ticket_to_zendesk(self, ticket): - response = requests.post( + response = self.requests_session.post( self.ZENDESK_TICKET_URL, json=ticket.request_data, auth=(f"{self.NOTIFY_ZENDESK_EMAIL}/token", self.api_key) ) @@ -91,7 +95,7 @@ def _upload_attachment(self, attachment: NotifySupportTicketAttachment): upload_url = self.ZENDESK_UPLOAD_FILE_URL + "?" + urlencode(query_params) - response = requests.post( + response = self.requests_session.post( upload_url, headers={"Content-Type": attachment.content_type}, data=attachment.filedata, @@ -137,7 +141,7 @@ def update_ticket( data["ticket"]["status"] = status.value update_url = self.ZENDESK_UPDATE_TICKET_URL.format(ticket_id=ticket_id) - response = requests.put( + response = self.requests_session.put( update_url, json=data, auth=(f"{self.NOTIFY_ZENDESK_EMAIL}/token", self.api_key), diff --git a/tests/clients/zendesk/test_zendesk_client.py b/tests/clients/zendesk/test_zendesk_client.py index 1a674fe0c..bb585d508 100644 --- a/tests/clients/zendesk/test_zendesk_client.py +++ b/tests/clients/zendesk/test_zendesk_client.py @@ -17,14 +17,8 @@ @pytest.fixture(scope="function") -def zendesk_client(app): - client = ZendeskClient() - - app.config["ZENDESK_API_KEY"] = "testkey" - - client.init_app(app) - - return client +def zendesk_client(): + return ZendeskClient(api_key="testkey") def test_zendesk_client_send_ticket_to_zendesk(zendesk_client, app, rmock, caplog): @@ -63,7 +57,7 @@ def test_zendesk_client_send_ticket_to_zendesk_error(zendesk_client, app, rmock, assert "Zendesk create ticket request failed with 401 '{'foo': 'bar'}'" in caplog.messages -def test_zendesk_client_send_ticket_to_zendesk_with_user_suspended_error(zendesk_client, rmock, caplog): +def test_zendesk_client_send_ticket_to_zendesk_with_user_suspended_error(zendesk_client, app, rmock, caplog): rmock.request( "POST", ZendeskClient.ZENDESK_TICKET_URL, From 2dec970f09497bb057055903b4e30ebc578705e6 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 23 Aug 2024 16:11:10 +0100 Subject: [PATCH 114/211] Bump version to 83.0.0 --- CHANGELOG.md | 5 +++++ notifications_utils/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0df7b1847..d4f89a83a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG +## 83.0.0 + +* `AntivirusClient` and `ZendeskClient` are no longer thread-safe because they now use persistent requests sessions. Thread-local instances should be used in place of any global instances in any situations where threading is liable to be used +* `AntivirusClient` and `ZendeskClient` have had their `init_app(...)` methods removed as this is an awkward pattern to use for initializing thread-local instances. It is recommended to use `LazyLocalGetter` to construct new instances on-demand, passing configuration parameters via the constructor arguments in a `factory` function. + ## 82.7.0 * Add `expected_type` mechanism to `LazyLocalGetter` diff --git a/notifications_utils/version.py b/notifications_utils/version.py index d00cbe4ec..8a9f3d64a 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "82.7.0" # b21ed1702bc409f764d6898e729430b7 +__version__ = "83.0.0" # 9ec4c0e5f2c033df86cfa33627e3c483 From 79d1674b98f7f316f00bb15b2cf08ae2466156e3 Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Thu, 12 Sep 2024 08:19:04 +0100 Subject: [PATCH 115/211] Updates phone_numbers library to 8.13.45 to fix a validation bug with certain Jersey phone numbers It was discovered that a subset of valid Jersey phone numbers were being incorrectly invalidated by the phone_numbers library code due to a bug in the regex phone numbers were being validated against for the Jersey region. This new code provides a fix --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- setup.py | 2 +- .../recipient_validation/test_phone_number.py | 18 ++++++++++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4f89a83a..c1cf14bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +# 83.0.1 + +* Updates phone_numbers to 8.13.45 to apply a fix to the metadata for phone numbers that was discovered causing a subset of valid Jersey numbers to be incorrectly invalidated + ## 83.0.0 * `AntivirusClient` and `ZendeskClient` are no longer thread-safe because they now use persistent requests sessions. Thread-local instances should be used in place of any global instances in any situations where threading is liable to be used diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 8a9f3d64a..9b4212453 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "83.0.0" # 9ec4c0e5f2c033df86cfa33627e3c483 +__version__ = "83.0.1" # b5ba071d02969864cf049d54563c94f6 diff --git a/setup.py b/setup.py index 56df70091..bdf95f9ae 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ "statsd>=4.0.1", "Flask-Redis>=0.4.0", "pyyaml>=6.0.1", - "phonenumbers>=8.13.18", + "phonenumbers>=8.13.45", "pytz>=2024.1", "smartypants>=2.0.1", "pypdf>=3.13.0", diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index 4dc663274..cd5dbccd5 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -46,6 +46,14 @@ "+43 676 111 222 333 4", # Austrian 13 digit phone numbers ] +# placeholder to be removed when the old validation code is removed +# categorising these numbers as international numbers will cause old checks +# for uk numbers to fail +valid_channel_island_numbers = [ + "+447797292290", # Jersey + "+447797333214", # Jersey +] + valid_mobile_phone_numbers = valid_uk_mobile_phone_numbers + valid_international_phone_numbers @@ -564,6 +572,16 @@ def test_international_does_not_normalise_to_uk_number(self, phone_number, expec PhoneNumber(phone_number, allow_international=False) assert exc.value.code == expected_error_code + # We discovered a bug with the phone_numbers library causing some valid JE numbers + # to evaluate as invalid. Realiably sending to Crown Dependencies is very important + # this test serves to alert us if a known failing edge case arises again. + @pytest.mark.parametrize("phone_number", valid_channel_island_numbers) + def test_channel_island_numbers_are_valid(self, phone_number): + try: + PhoneNumber(phone_number, allow_international=True) + except InvalidPhoneError: + pytest.fail("Unexpected InvalidPhoneError") + def test_empty_phone_number_is_rejected_with_correct_v2_error_message(): phone_number = "" From 259c8e904858e0adeec7ab42c690feb4e4d9822a Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 12 Sep 2024 12:40:39 +0100 Subject: [PATCH 116/211] Revert "ZendeskClient: use a persistent requests session" This reverts commit 08227b2e988dbad60fdb27192f0d52e61341b225. --- .../clients/zendesk/zendesk_client.py | 20 ++++++++----------- tests/clients/zendesk/test_zendesk_client.py | 12 ++++++++--- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/notifications_utils/clients/zendesk/zendesk_client.py b/notifications_utils/clients/zendesk/zendesk_client.py index 35b09354e..c3d3f0a5a 100644 --- a/notifications_utils/clients/zendesk/zendesk_client.py +++ b/notifications_utils/clients/zendesk/zendesk_client.py @@ -48,12 +48,6 @@ class NotifySupportTicketComment: class ZendeskClient: - """ - A client for the Zendesk API - - This class is not thread-safe. - """ - # the account used to authenticate with. If no requester is provided, the ticket will come from this account. NOTIFY_ZENDESK_EMAIL = "zd-api-notify@digital.cabinet-office.gov.uk" @@ -61,12 +55,14 @@ class ZendeskClient: ZENDESK_UPDATE_TICKET_URL = "https://govuk.zendesk.com/api/v2/tickets/{ticket_id}" ZENDESK_UPLOAD_FILE_URL = "https://govuk.zendesk.com/api/v2/uploads.json" - def __init__(self, api_key): - self.api_key = api_key - self.requests_session = requests.Session() + def __init__(self): + self.api_key = None + + def init_app(self, app, *args, **kwargs): + self.api_key = app.config.get("ZENDESK_API_KEY") def send_ticket_to_zendesk(self, ticket): - response = self.requests_session.post( + response = requests.post( self.ZENDESK_TICKET_URL, json=ticket.request_data, auth=(f"{self.NOTIFY_ZENDESK_EMAIL}/token", self.api_key) ) @@ -95,7 +91,7 @@ def _upload_attachment(self, attachment: NotifySupportTicketAttachment): upload_url = self.ZENDESK_UPLOAD_FILE_URL + "?" + urlencode(query_params) - response = self.requests_session.post( + response = requests.post( upload_url, headers={"Content-Type": attachment.content_type}, data=attachment.filedata, @@ -141,7 +137,7 @@ def update_ticket( data["ticket"]["status"] = status.value update_url = self.ZENDESK_UPDATE_TICKET_URL.format(ticket_id=ticket_id) - response = self.requests_session.put( + response = requests.put( update_url, json=data, auth=(f"{self.NOTIFY_ZENDESK_EMAIL}/token", self.api_key), diff --git a/tests/clients/zendesk/test_zendesk_client.py b/tests/clients/zendesk/test_zendesk_client.py index bb585d508..1a674fe0c 100644 --- a/tests/clients/zendesk/test_zendesk_client.py +++ b/tests/clients/zendesk/test_zendesk_client.py @@ -17,8 +17,14 @@ @pytest.fixture(scope="function") -def zendesk_client(): - return ZendeskClient(api_key="testkey") +def zendesk_client(app): + client = ZendeskClient() + + app.config["ZENDESK_API_KEY"] = "testkey" + + client.init_app(app) + + return client def test_zendesk_client_send_ticket_to_zendesk(zendesk_client, app, rmock, caplog): @@ -57,7 +63,7 @@ def test_zendesk_client_send_ticket_to_zendesk_error(zendesk_client, app, rmock, assert "Zendesk create ticket request failed with 401 '{'foo': 'bar'}'" in caplog.messages -def test_zendesk_client_send_ticket_to_zendesk_with_user_suspended_error(zendesk_client, app, rmock, caplog): +def test_zendesk_client_send_ticket_to_zendesk_with_user_suspended_error(zendesk_client, rmock, caplog): rmock.request( "POST", ZendeskClient.ZENDESK_TICKET_URL, From e7f981432de90f8c18de20e4eb332c5161176a48 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 12 Sep 2024 12:42:24 +0100 Subject: [PATCH 117/211] Revert "AntivirusClient: use a persistent requests session" This reverts commit 12e9aa6b3b48e61a632209cb8245e4a01417e195. --- .../clients/antivirus/antivirus_client.py | 13 +++++-------- tests/clients/antivirus/test_antivirus_client.py | 8 ++++---- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/notifications_utils/clients/antivirus/antivirus_client.py b/notifications_utils/clients/antivirus/antivirus_client.py index 74a165af3..f723db310 100644 --- a/notifications_utils/clients/antivirus/antivirus_client.py +++ b/notifications_utils/clients/antivirus/antivirus_client.py @@ -21,16 +21,13 @@ def from_exception(cls, e): class AntivirusClient: - """ - A client for the antivirus API - - This class is not thread-safe. - """ - def __init__(self, api_host=None, auth_token=None): self.api_host = api_host self.auth_token = auth_token - self.requests_session = requests.Session() + + def init_app(self, app): + self.api_host = app.config["ANTIVIRUS_API_HOST"] + self.auth_token = app.config["ANTIVIRUS_API_KEY"] def scan(self, document_stream): headers = {"Authorization": f"Bearer {self.auth_token}"} @@ -38,7 +35,7 @@ def scan(self, document_stream): headers.update(request.get_onwards_request_headers()) try: - response = self.requests_session.post( + response = requests.post( f"{self.api_host}/scan", headers=headers, files={"document": document_stream}, diff --git a/tests/clients/antivirus/test_antivirus_client.py b/tests/clients/antivirus/test_antivirus_client.py index f5bab891a..173ac0152 100644 --- a/tests/clients/antivirus/test_antivirus_client.py +++ b/tests/clients/antivirus/test_antivirus_client.py @@ -12,10 +12,10 @@ @pytest.fixture(scope="function") def app_antivirus_client(app, mocker): - client = AntivirusClient( - api_host="https://antivirus", - auth_token="test-antivirus-key", - ) + client = AntivirusClient() + app.config["ANTIVIRUS_API_HOST"] = "https://antivirus" + app.config["ANTIVIRUS_API_KEY"] = "test-antivirus-key" + client.init_app(app) return app, client From 16c09a0b6863af78edfd567a016ac0014da1c24e Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 12 Sep 2024 12:45:57 +0100 Subject: [PATCH 118/211] Bump version to 84.0.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1cf14bd9..831a63081 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 84.0.0 + +* `AntivirusClient` and `ZendeskClient` have returned to their behaviour as of 82.x.x to allow the 83.0.1 fix to go out to apps without the required changes. + # 83.0.1 * Updates phone_numbers to 8.13.45 to apply a fix to the metadata for phone numbers that was discovered causing a subset of valid Jersey numbers to be incorrectly invalidated diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 9b4212453..0ad75677d 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "83.0.1" # b5ba071d02969864cf049d54563c94f6 +__version__ = "84.0.0" # e8765b4f80e01f From 886f6f4827d3ab55a86f426d1f1b3061d781cdc1 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 23 Aug 2024 09:45:35 +0100 Subject: [PATCH 119/211] =?UTF-8?q?Drop=20Girobank=E2=80=99s=20postcode=20?= =?UTF-8?q?from=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://www.poweredbypaf.com/wp-content/uploads/2017/07/Latest-Programmers_guide_Edition-7-Version-6.pdf --- CHANGELOG.md | 4 ++++ notifications_utils/recipient_validation/postal_address.py | 3 +-- notifications_utils/version.py | 2 +- tests/recipient_validation/test_postal_address.py | 5 ++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 831a63081..aee785ddd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 84.0.1 + +* Remove GIR 0AA from valid postcodes + ## 84.0.0 * `AntivirusClient` and `ZendeskClient` have returned to their behaviour as of 82.x.x to allow the 83.0.1 fix to go out to apps without the required changes. diff --git a/notifications_utils/recipient_validation/postal_address.py b/notifications_utils/recipient_validation/postal_address.py index 08d0acaaa..82b30da29 100644 --- a/notifications_utils/recipient_validation/postal_address.py +++ b/notifications_utils/recipient_validation/postal_address.py @@ -243,8 +243,7 @@ def normalise_postcode(postcode): def _is_a_real_uk_postcode(postcode): normalised = normalise_postcode(postcode) - # GIR0AA is Girobank - pattern = re.compile(rf"(({'|'.join(UK_POSTCODE_ZONES)})[0-9][0-9A-Z]?[0-9][A-BD-HJLNP-UW-Z]{{2}})|(GIR0AA)") + pattern = re.compile(rf"(({'|'.join(UK_POSTCODE_ZONES)})[0-9][0-9A-Z]?[0-9][A-BD-HJLNP-UW-Z]{{2}})") return bool(pattern.fullmatch(normalised)) diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 0ad75677d..34222e5b2 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "84.0.0" # e8765b4f80e01f +__version__ = "84.0.1" # 87c24ae9785f1425177135de028b83e8 diff --git a/tests/recipient_validation/test_postal_address.py b/tests/recipient_validation/test_postal_address.py index 2a96877c2..7534ce882 100644 --- a/tests/recipient_validation/test_postal_address.py +++ b/tests/recipient_validation/test_postal_address.py @@ -718,9 +718,8 @@ def test_normalise_postcode(postcode, normalised_postcode): ("BF1 3AA", True), ("BF13AA", True), (" BF2 0FR ", True), - # Giro Bank valid postcode and invalid postcode - ("GIR0AA", True), - ("GIR0AB", False), + # Giro Bank’s vanity postcode is deprecated + ("GIR0AA", False), # Gibraltar’s one postcode is not valid because it’s in the # Europe postal zone ("GX111AA", False), From 484d4530885448ef4a61ce811bcefa015bc3e941 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 16 Sep 2024 13:57:19 +0100 Subject: [PATCH 120/211] Automatically copy config after version bump A new version of utils might have introduced new requirements, or new linter options. So to keep the config in sync with the version of utils specified in the requirements file we should automatically check for new config files every time we bump the version in an app. --- notifications_utils/version_tools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/notifications_utils/version_tools.py b/notifications_utils/version_tools.py index 42d0a80d8..734ec13d8 100644 --- a/notifications_utils/version_tools.py +++ b/notifications_utils/version_tools.py @@ -31,6 +31,8 @@ def upgrade_version(): write_version_to_requirements_file(newest_version) + copy_config() + print( # noqa: T201 f"{color.GREEN}✅ {color.BOLD}notifications-utils bumped to {newest_version}{color.END}\n\n" f"{color.YELLOW}{color.UNDERLINE}Now run:{color.END}\n\n" From 2eb898ecbb2a745e0fb5510b1e43eb490f2e95a4 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Tue, 17 Sep 2024 15:34:25 +0100 Subject: [PATCH 121/211] Add a new custom field to the Zendesk client The Zendesk client now takes an optional argument called `user_created_at`. If provided, this value is used to populate a new field we have on the form which is a calendar date picker. This will let us analyse how long the people opening tickets have been using Notify for. --- .../clients/zendesk/zendesk_client.py | 21 ++++++++++ tests/clients/zendesk/test_zendesk_client.py | 40 +++++++++++++++++-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/notifications_utils/clients/zendesk/zendesk_client.py b/notifications_utils/clients/zendesk/zendesk_client.py index c3d3f0a5a..12cf0fd8c 100644 --- a/notifications_utils/clients/zendesk/zendesk_client.py +++ b/notifications_utils/clients/zendesk/zendesk_client.py @@ -4,6 +4,7 @@ import typing from urllib.parse import urlencode +import pytz import requests from flask import current_app @@ -190,6 +191,7 @@ def __init__( service_id=None, email_ccs=None, message_as_html=False, + user_created_at=None, ): self.subject = subject self.message = message @@ -205,6 +207,7 @@ def __init__( self.service_id = service_id self.email_ccs = email_ccs self.message_as_html = message_as_html + self.user_created_at = user_created_at @property def request_data(self): @@ -234,6 +237,20 @@ def request_data(self): return data + def _format_user_created_at_value(self, created_at_datetime: None | datetime.datetime) -> None | str: + """ + If given a UTC datetime this returns the date as a string in the format of "YYYY-MM-DD" for use + with the Zendesk calendar picker + """ + if created_at_datetime: + user_created_at_as_london_datetime = created_at_datetime.astimezone(pytz.timezone("Europe/London")) + + formatted_date = user_created_at_as_london_datetime.strftime("%Y-%m-%d") + + return formatted_date + + return None + def _get_custom_fields(self): org_type_tag = f"notify_org_type_{self.org_type}" if self.org_type else None custom_fields = [ @@ -241,6 +258,10 @@ def _get_custom_fields(self): {"id": "360022943959", "value": self.org_id}, # Notify Organisation ID field {"id": "360022943979", "value": org_type_tag}, # Notify Organisation type field {"id": "1900000745014", "value": self.service_id}, # Notify Service ID field + { + "id": "15925693889308", + "value": self._format_user_created_at_value(self.user_created_at), + }, # Notify user account creation date field ] if self.notify_ticket_type: diff --git a/tests/clients/zendesk/test_zendesk_client.py b/tests/clients/zendesk/test_zendesk_client.py index 1a674fe0c..a30b2317e 100644 --- a/tests/clients/zendesk/test_zendesk_client.py +++ b/tests/clients/zendesk/test_zendesk_client.py @@ -129,6 +129,7 @@ def test_notify_support_ticket_request_data(p1_arg, expected_tags, expected_prio {"id": "360022943959", "value": None}, {"id": "360022943979", "value": None}, {"id": "1900000745014", "value": None}, + {"id": "15925693889308", "value": None}, ], } } @@ -151,9 +152,17 @@ def test_notify_support_ticket_request_data_with_user_name_and_email(name, zende @pytest.mark.parametrize( - "custom_fields, tech_ticket_tag, notify_task_type, org_id, org_type, service_id", + "custom_fields, tech_ticket_tag, notify_task_type, org_id, org_type, service_id, user_created_at", [ - ({"notify_ticket_type": NotifyTicketType.TECHNICAL}, "notify_ticket_type_technical", None, None, None, None), + ( + {"notify_ticket_type": NotifyTicketType.TECHNICAL}, + "notify_ticket_type_technical", + None, + None, + None, + None, + None, + ), ( {"notify_ticket_type": NotifyTicketType.NON_TECHNICAL}, "notify_ticket_type_non_technical", @@ -161,6 +170,7 @@ def test_notify_support_ticket_request_data_with_user_name_and_email(name, zende None, None, None, + None, ), ( {"notify_task_type": "notify_task_email_branding"}, @@ -169,14 +179,16 @@ def test_notify_support_ticket_request_data_with_user_name_and_email(name, zende None, None, None, + None, ), ( - {"org_id": "1234", "org_type": "local"}, + {"org_id": "1234", "org_type": "local", "user_created_at": datetime.datetime(2024, 10, 10, 12, 36)}, None, None, "1234", "notify_org_type_local", None, + "2024-10-10", ), ( {"service_id": "abcd", "org_type": "nhs"}, @@ -185,6 +197,7 @@ def test_notify_support_ticket_request_data_with_user_name_and_email(name, zende None, "notify_org_type_nhs", "abcd", + None, ), ], ) @@ -195,6 +208,7 @@ def test_notify_support_ticket_request_data_custom_fields( org_id, org_type, service_id, + user_created_at, ): notify_ticket_form = NotifySupportTicket("subject", "message", "question", **custom_fields) @@ -208,6 +222,9 @@ def test_notify_support_ticket_request_data_custom_fields( assert {"id": "360022943959", "value": org_id} in notify_ticket_form.request_data["ticket"]["custom_fields"] assert {"id": "360022943979", "value": org_type} in notify_ticket_form.request_data["ticket"]["custom_fields"] assert {"id": "1900000745014", "value": service_id} in notify_ticket_form.request_data["ticket"]["custom_fields"] + assert {"id": "15925693889308", "value": user_created_at} in notify_ticket_form.request_data["ticket"][ + "custom_fields" + ] def test_notify_support_ticket_request_data_email_ccs(): @@ -239,11 +256,28 @@ def test_notify_support_ticket_with_html_body(): {"id": "360022943959", "value": None}, {"id": "360022943979", "value": None}, {"id": "1900000745014", "value": None}, + {"id": "15925693889308", "value": None}, ], } } +@pytest.mark.parametrize( + "user_created_at, expected_value", + [ + (None, None), + (datetime.datetime(2023, 11, 7, 8, 34, 54, tzinfo=datetime.UTC), "2023-11-07"), + (datetime.datetime(2023, 11, 7, 23, 34, 54, tzinfo=datetime.UTC), "2023-11-07"), + (datetime.datetime(2023, 6, 7, 23, 34, 54, tzinfo=datetime.UTC), "2023-06-08"), + (datetime.datetime(2023, 6, 7, 12, 34, 54, tzinfo=datetime.UTC), "2023-06-07"), + ], +) +def test_notify_support_ticket__format_user_created_at_value(user_created_at, expected_value): + notify_ticket_form = NotifySupportTicket("subject", "message", "task") + + assert notify_ticket_form._format_user_created_at_value(user_created_at) == expected_value + + class TestZendeskClientUploadAttachment: def test_upload_csv(self, zendesk_client, app, rmock): rmock.request( From 468057f387141cca148188c826a6bd8639ab53b4 Mon Sep 17 00:00:00 2001 From: Katie Smith Date: Tue, 17 Sep 2024 15:36:12 +0100 Subject: [PATCH 122/211] Minor version bump to 84.2.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b42fb1f5f..dec9c9b73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 84.2.0 + +* The Zendesk client takes a new optional argument, `user_created_at` which populates a new field on the Notify Zendesk form if provided. + ## 84.1.1 * Remove GIR 0AA from valid postcodes diff --git a/notifications_utils/version.py b/notifications_utils/version.py index b26ff0ce4..002a0f61b 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "84.1.1" # 87c24ae9785f1425177135de028b83e8 +__version__ = "84.2.0" # 6c167f6c73b099d58fdd5952e7c237b0 From 390e0833ed5677372d85d52d77ad133240e63b84 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 9 Sep 2024 17:12:00 +0100 Subject: [PATCH 123/211] Migrate asset fingerprinter from utils MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So we can shared it between our frontend apps. This continues its journey, having been: - introduced here: https://github.com/Crown-Commercial-Service/digitalmarketplace-buyer-frontend/pull/111/files - refactored into shared code here: https://github.com/Crown-Commercial-Service/digitalmarketplace-utils/pull/102/files - copied into Notify here: https://github.com/alphagov/notifications-admin/pull/171/files (“They have it in a shared codebase, we only have one frontend app so don’t need to do that.”) - copied into Document Download here: https://github.com/alphagov/document-download-frontend/pull/1/files --- notifications_utils/asset_fingerprinter.py | 45 ++++++++++ tests/test_asset_fingerprinter.py | 98 ++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 notifications_utils/asset_fingerprinter.py create mode 100644 tests/test_asset_fingerprinter.py diff --git a/notifications_utils/asset_fingerprinter.py b/notifications_utils/asset_fingerprinter.py new file mode 100644 index 000000000..22ad1c519 --- /dev/null +++ b/notifications_utils/asset_fingerprinter.py @@ -0,0 +1,45 @@ +import hashlib + + +class AssetFingerprinter: + """ + Get a unique hash for an asset file, so that it doesn't stay cached + when it changes + + Usage: + + in the application + template_data.asset_fingerprinter = AssetFingerprinter() + + where template data is how you pass variables to every template. + + in template.html: + {{ asset_fingerprinter.get_url('stylesheets/application.css') }} + + * 'app/static' is assumed to be the root for all asset files + """ + + def __init__(self, asset_root="/static/", filesystem_path="app/static/"): + self._cache = {} + self._asset_root = asset_root + self._filesystem_path = filesystem_path + + def get_url(self, asset_path, with_querystring_hash=True): + if not with_querystring_hash: + return self._asset_root + asset_path + if asset_path not in self._cache: + self._cache[asset_path] = ( + self._asset_root + asset_path + "?" + self.get_asset_fingerprint(self._filesystem_path + asset_path) + ) + return self._cache[asset_path] + + def get_asset_fingerprint(self, asset_file_path): + return hashlib.md5(self.get_asset_file_contents(asset_file_path)).hexdigest() + + def get_asset_file_contents(self, asset_file_path): + with open(asset_file_path, "rb") as asset_file: + contents = asset_file.read() + return contents + + +asset_fingerprinter = AssetFingerprinter() diff --git a/tests/test_asset_fingerprinter.py b/tests/test_asset_fingerprinter.py new file mode 100644 index 000000000..2fa37ad07 --- /dev/null +++ b/tests/test_asset_fingerprinter.py @@ -0,0 +1,98 @@ +from notifications_utils.asset_fingerprinter import AssetFingerprinter + + +class TestAssetFingerprint: + def test_url_format(self, mocker): + get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") + get_file_content_mock.return_value = """ + body { + font-family: nta; + } + """.encode( + "utf-8" + ) + asset_fingerprinter = AssetFingerprinter(asset_root="/suppliers/static/") + assert ( + asset_fingerprinter.get_url("application.css") + == "/suppliers/static/application.css?418e6f4a6cdf1142e45c072ed3e1c90a" # noqa + ) + assert ( + asset_fingerprinter.get_url("application-ie6.css") + == "/suppliers/static/application-ie6.css?418e6f4a6cdf1142e45c072ed3e1c90a" # noqa + ) + + def test_building_file_path(self, mocker): + get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") + get_file_content_mock.return_value = """ + document.write('Hello world!'); + """.encode( + "utf-8" + ) + fingerprinter = AssetFingerprinter() + fingerprinter.get_url("javascripts/application.js") + fingerprinter.get_asset_file_contents.assert_called_with("app/static/javascripts/application.js") + + def test_hashes_are_consistent(self, mocker): + get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") + get_file_content_mock.return_value = """ + body { + font-family: nta; + } + """.encode( + "utf-8" + ) + asset_fingerprinter = AssetFingerprinter() + assert asset_fingerprinter.get_asset_fingerprint( + "application.css" + ) == asset_fingerprinter.get_asset_fingerprint("same_contents.css") + + def test_hashes_are_different_for_different_files(self, mocker): + get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") + asset_fingerprinter = AssetFingerprinter() + get_file_content_mock.return_value = """ + body { + font-family: nta; + } + """.encode( + "utf-8" + ) + css_hash = asset_fingerprinter.get_asset_fingerprint("application.css") + get_file_content_mock.return_value = """ + document.write('Hello world!'); + """.encode( + "utf-8" + ) + js_hash = asset_fingerprinter.get_asset_fingerprint("application.js") + assert js_hash != css_hash + + def test_hash_gets_cached(self, mocker): + get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") + get_file_content_mock.return_value = """ + body { + font-family: nta; + } + """.encode( + "utf-8" + ) + fingerprinter = AssetFingerprinter() + assert fingerprinter.get_url("application.css") == "/static/application.css?418e6f4a6cdf1142e45c072ed3e1c90a" + fingerprinter._cache["application.css"] = "a1a1a1" + assert fingerprinter.get_url("application.css") == "a1a1a1" + fingerprinter.get_asset_file_contents.assert_called_once_with("app/static/application.css") + + def test_without_hash_if_requested(self, mocker): + fingerprinter = AssetFingerprinter() + assert ( + fingerprinter.get_url( + "application.css", + with_querystring_hash=False, + ) + == "/static/application.css" + ) + assert fingerprinter._cache == {} + + +class TestAssetFingerprintWithUnicode: + def test_can_read_self(self): + "Ralph’s apostrophe is a string containing a unicode character" + AssetFingerprinter(filesystem_path="tests/").get_url("test_asset_fingerprinter.py") From 998a6b2338425f7df9322306aa321fbd3a2b548a Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 9 Sep 2024 17:17:44 +0100 Subject: [PATCH 124/211] Use Pathlib to read files contents Pathlib is a more modern way of dealing with files, and its syntax is a bit cleaner. --- notifications_utils/asset_fingerprinter.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/notifications_utils/asset_fingerprinter.py b/notifications_utils/asset_fingerprinter.py index 22ad1c519..42008c596 100644 --- a/notifications_utils/asset_fingerprinter.py +++ b/notifications_utils/asset_fingerprinter.py @@ -1,4 +1,5 @@ import hashlib +import pathlib class AssetFingerprinter: @@ -37,9 +38,7 @@ def get_asset_fingerprint(self, asset_file_path): return hashlib.md5(self.get_asset_file_contents(asset_file_path)).hexdigest() def get_asset_file_contents(self, asset_file_path): - with open(asset_file_path, "rb") as asset_file: - contents = asset_file.read() - return contents + return pathlib.Path(asset_file_path).read_bytes() asset_fingerprinter = AssetFingerprinter() From a41815b3bb6717f9c350598d01f8dc7dca46c29e Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 9 Sep 2024 17:36:28 +0100 Subject: [PATCH 125/211] Appease Ruff UP012 [*] Unnecessary call to `encode` as UTF-8 --- tests/test_asset_fingerprinter.py | 36 +++++++++++-------------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/tests/test_asset_fingerprinter.py b/tests/test_asset_fingerprinter.py index 2fa37ad07..96b367370 100644 --- a/tests/test_asset_fingerprinter.py +++ b/tests/test_asset_fingerprinter.py @@ -4,13 +4,11 @@ class TestAssetFingerprint: def test_url_format(self, mocker): get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") - get_file_content_mock.return_value = """ + get_file_content_mock.return_value = b""" body { font-family: nta; } - """.encode( - "utf-8" - ) + """ asset_fingerprinter = AssetFingerprinter(asset_root="/suppliers/static/") assert ( asset_fingerprinter.get_url("application.css") @@ -23,24 +21,20 @@ def test_url_format(self, mocker): def test_building_file_path(self, mocker): get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") - get_file_content_mock.return_value = """ + get_file_content_mock.return_value = b""" document.write('Hello world!'); - """.encode( - "utf-8" - ) + """ fingerprinter = AssetFingerprinter() fingerprinter.get_url("javascripts/application.js") fingerprinter.get_asset_file_contents.assert_called_with("app/static/javascripts/application.js") def test_hashes_are_consistent(self, mocker): get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") - get_file_content_mock.return_value = """ + get_file_content_mock.return_value = b""" body { font-family: nta; } - """.encode( - "utf-8" - ) + """ asset_fingerprinter = AssetFingerprinter() assert asset_fingerprinter.get_asset_fingerprint( "application.css" @@ -49,31 +43,25 @@ def test_hashes_are_consistent(self, mocker): def test_hashes_are_different_for_different_files(self, mocker): get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") asset_fingerprinter = AssetFingerprinter() - get_file_content_mock.return_value = """ + get_file_content_mock.return_value = b""" body { font-family: nta; } - """.encode( - "utf-8" - ) + """ css_hash = asset_fingerprinter.get_asset_fingerprint("application.css") - get_file_content_mock.return_value = """ + get_file_content_mock.return_value = b""" document.write('Hello world!'); - """.encode( - "utf-8" - ) + """ js_hash = asset_fingerprinter.get_asset_fingerprint("application.js") assert js_hash != css_hash def test_hash_gets_cached(self, mocker): get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") - get_file_content_mock.return_value = """ + get_file_content_mock.return_value = b""" body { font-family: nta; } - """.encode( - "utf-8" - ) + """ fingerprinter = AssetFingerprinter() assert fingerprinter.get_url("application.css") == "/static/application.css?418e6f4a6cdf1142e45c072ed3e1c90a" fingerprinter._cache["application.css"] = "a1a1a1" From 16c1b916535eaf5225398835ca68683b9c84b6ff Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Mon, 9 Sep 2024 17:19:25 +0100 Subject: [PATCH 126/211] Minor version bump --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dec9c9b73..0abc5faf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 84.3.0 + +* Adds `asset_fingerprinter.AssetFingerprinter` to replace the versions duplicated across our frontend apps + ## 84.2.0 * The Zendesk client takes a new optional argument, `user_created_at` which populates a new field on the Notify Zendesk form if provided. diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 002a0f61b..d2c3a6592 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "84.2.0" # 6c167f6c73b099d58fdd5952e7c237b0 +__version__ = "84.3.0" # 37dac8b3e2812ede962df582c8fe48ec From 3f981081f9fc91d864686332fdfe647fb57a96d5 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 19 Sep 2024 12:27:17 +0100 Subject: [PATCH 127/211] Revert "Bump test requirements to the latest versions" This reverts commit d72195a650ec4cd63a689a1d6c57cd0c599a4398. These new versions were not compatible with existing dependencies in the API --- .pre-commit-config.yaml | 4 ++-- requirements_for_test_common.txt | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3426777b8..be6f150c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,12 +7,12 @@ repos: - id: check-yaml - id: debug-statements - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.6.4' + rev: 'v0.3.7' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black - rev: 24.8.0 + rev: 24.4.0 hooks: - id: black name: black (python) diff --git a/requirements_for_test_common.txt b/requirements_for_test_common.txt index 88bd3c6f4..4e69f1f33 100644 --- a/requirements_for_test_common.txt +++ b/requirements_for_test_common.txt @@ -1,12 +1,12 @@ -beautifulsoup4==4.12.3 -pytest==8.3.2 -pytest-env==1.1.3 -pytest-mock==3.14.0 -pytest-xdist==3.6.1 -pytest-testmon==2.1.1 +beautifulsoup4==4.11.1 +pytest==7.2.0 +pytest-env==0.8.1 +pytest-mock==3.9.0 +pytest-xdist==3.0.2 +pytest-testmon==2.1.0 pytest-watch==4.2.0 -requests-mock==1.12.1 -freezegun==1.5.1 +requests-mock==1.10.0 +freezegun==1.2.2 -black==24.8.0 # Also update `.pre-commit-config.yaml` if this changes -ruff==0.6.4 # Also update `.pre-commit-config.yaml` if this changes +black==24.4.0 # Also update `.pre-commit-config.yaml` if this changes +ruff==0.3.7 # Also update `.pre-commit-config.yaml` if this changes From 52fa4aa9856c93bee87c9c99828fbb53bf50ce98 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Thu, 19 Sep 2024 12:28:59 +0100 Subject: [PATCH 128/211] Minor version bump --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dec9c9b73..96e4188fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 84.3.0 + +* Reverts 84.1.0 + ## 84.2.0 * The Zendesk client takes a new optional argument, `user_created_at` which populates a new field on the Notify Zendesk form if provided. diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 002a0f61b..e04ae45b3 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "84.2.0" # 6c167f6c73b099d58fdd5952e7c237b0 +__version__ = "84.3.0" # e8f0e1b3e2ed73b2e0962487a0292e38 From 305010ec962ef53fddd4049dcfeca30c404f247a Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Wed, 21 Aug 2024 16:55:20 +0100 Subject: [PATCH 129/211] Added allow_landline argument to PhoneNumber constructor The default behaviour of notifications_utils.recipient_validation.phone_number.PhoneNumber validates landline numbers. We want to be able to toggle this on or off via a constructor argument so that this code can be used to validate phone numbers for all services, not just those with the sms_to_uk_landline permission. This change adds the constructor argument allow_landline and makes changes to the validation code to throw an exception if a valid landline is passed when allow_landline=False. --- CHANGELOG.md | 4 ++++ .../recipient_validation/phone_number.py | 18 +++++++++++++++++- notifications_utils/recipients.py | 11 +++++++++-- .../recipient_validation/test_phone_number.py | 14 ++++++++++---- 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96e4188fc..67921416f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 85.0.0 + +* BREAKING CHANGE: The `Phonenumber` class now accepts a flag `allow_landline`, which defaults to False. This changes the previous default behaviour, allowing landlines. + ## 84.3.0 * Reverts 84.1.0 diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index 118d46b8d..cf68d4349 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -16,6 +16,12 @@ UK_PREFIX = "44" +LANDLINE_CODES = { + phonenumbers.PhoneNumberType.FIXED_LINE, + phonenumbers.PhoneNumberType.FIXED_LINE_OR_MOBILE, + phonenumbers.PhoneNumberType.UAN, +} + ALLOW_LIST = { phonenumbers.PhoneNumberType.FIXED_LINE, phonenumbers.PhoneNumberType.MOBILE, @@ -174,14 +180,16 @@ class PhoneNumber: international phone numbers to be allowed or not. """ - def __init__(self, phone_number: str, *, allow_international: bool) -> None: + def __init__(self, phone_number: str, *, allow_international: bool, allow_landline: bool = False) -> None: self.raw_input = phone_number self.allow_international = allow_international + self.allow_landline = allow_landline try: self.number = self.validate_phone_number(phone_number) except InvalidPhoneError: phone_number = self._thoroughly_normalise_number(phone_number) self.number = self.validate_phone_number(phone_number) + self._raise_if_service_cannot_send_to_uk_landline_but_tries_to(phone_number) self._raise_if_service_cannot_send_to_international_but_tries_to(phone_number) def _raise_if_service_cannot_send_to_international_but_tries_to(self, phone_number): @@ -189,6 +197,14 @@ def _raise_if_service_cannot_send_to_international_but_tries_to(self, phone_numb if not self.allow_international and str(number.country_code) != UK_PREFIX: raise InvalidPhoneError(code=InvalidPhoneError.Codes.NOT_A_UK_MOBILE) + def _raise_if_service_cannot_send_to_uk_landline_but_tries_to(self, phone_number): + phone_number = self._try_parse_number(phone_number) + if phone_number.country_code != 44: + return + is_landline = phonenumbers.number_type(phone_number) in LANDLINE_CODES + if not self.allow_landline and is_landline: + raise InvalidPhoneError(code=InvalidPhoneError.Codes.NOT_A_UK_MOBILE) + @staticmethod def _try_parse_number(phone_number): try: diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index fa13021d5..85449d088 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -321,9 +321,16 @@ def _get_error_for_field(self, key, value): # noqa: C901 email_address.validate_email_address(value) if self.template_type == "sms": if self.allow_sms_to_uk_landline: - PhoneNumber(value, allow_international=self.allow_international_sms) + PhoneNumber( + value, + allow_international=self.allow_international_sms, + allow_landline=self.allow_sms_to_uk_landline, + ) else: - phone_number.validate_phone_number(value, international=self.allow_international_sms) + phone_number.validate_phone_number( + value, + international=self.allow_international_sms, + ) except InvalidRecipientError as error: return str(error) diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index cd5dbccd5..c0169e803 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -410,7 +410,7 @@ def test_rejects_invalid_uk_mobile_phone_numbers(self, phone_number, error_messa @pytest.mark.parametrize("phone_number", invalid_uk_landlines) def test_rejects_invalid_uk_landlines(self, phone_number): with pytest.raises(InvalidPhoneError) as e: - PhoneNumber(phone_number, allow_international=False) + PhoneNumber(phone_number, allow_international=False, allow_landline=True) assert e.value.code == InvalidPhoneError.Codes.INVALID_NUMBER @pytest.mark.parametrize( @@ -437,7 +437,13 @@ def test_allows_valid_international_phone_numbers(self, phone_number): @pytest.mark.parametrize("phone_number", valid_uk_landlines) def test_allows_valid_uk_landlines(self, phone_number): - assert PhoneNumber(phone_number, allow_international=True).is_uk_phone_number() is True + assert PhoneNumber(phone_number, allow_international=True, allow_landline=True).is_uk_phone_number() is True + + @pytest.mark.parametrize("phone_number", valid_uk_landlines) + def test_rejects_valid_uk_landlines_if_allow_landline_is_false(self, phone_number): + with pytest.raises(InvalidPhoneError) as exc: + PhoneNumber(phone_number, allow_international=True, allow_landline=False) + assert exc.value.code == InvalidPhoneError.Codes.NOT_A_UK_MOBILE @pytest.mark.parametrize("phone_number, expected_info", international_phone_info_fixtures) def test_get_international_phone_info(self, phone_number, expected_info): @@ -516,7 +522,7 @@ def test_get_human_readable_format(self, phone_number, expected_formatted): ], ) def test_validate_normalised_succeeds(self, phone_number, expected_normalised_number): - normalised_number = PhoneNumber(phone_number, allow_international=True) + normalised_number = PhoneNumber(phone_number, allow_international=True, allow_landline=True) assert str(normalised_number) == expected_normalised_number # TODO: decide if all these tests are useful to have. @@ -569,7 +575,7 @@ def test_tv_number_passes(self, phone_number, expected_valid_number): ) def test_international_does_not_normalise_to_uk_number(self, phone_number, expected_error_code): with pytest.raises(InvalidPhoneError) as exc: - PhoneNumber(phone_number, allow_international=False) + PhoneNumber(phone_number, allow_international=False, allow_landline=True) assert exc.value.code == expected_error_code # We discovered a bug with the phone_numbers library causing some valid JE numbers From 316b33fadfc69ad5eb08e77203055fd911c0eaab Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 20 Sep 2024 09:23:41 +0100 Subject: [PATCH 130/211] =?UTF-8?q?Don=E2=80=99t=20require=20implementatio?= =?UTF-8?q?n=20of=20`ALLOWED=5FPROPERTIES`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are going to replace this with annotations. Removing it from the abstract class means we won’t get an exception when we later remove it from the implementations in the tests. --- notifications_utils/serialised_model.py | 7 +------ tests/test_serialised_model.py | 27 ------------------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/notifications_utils/serialised_model.py b/notifications_utils/serialised_model.py index 8e925008e..27c368e91 100644 --- a/notifications_utils/serialised_model.py +++ b/notifications_utils/serialised_model.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod -class SerialisedModel(ABC): +class SerialisedModel: """ A SerialisedModel takes a dictionary, typically created by serialising a database object. It then takes the value of specified @@ -20,11 +20,6 @@ class SerialisedModel(ABC): ALLOWED_PROPERTIES list. """ - @property - @abstractmethod - def ALLOWED_PROPERTIES(self): - pass - def __init__(self, _dict): for property in self.ALLOWED_PROPERTIES: setattr(self, property, _dict[property]) diff --git a/tests/test_serialised_model.py b/tests/test_serialised_model.py index 18bbd91f7..fa84ddc40 100644 --- a/tests/test_serialised_model.py +++ b/tests/test_serialised_model.py @@ -9,36 +9,9 @@ def test_cant_be_instatiated_with_abstract_properties(): - class Custom(SerialisedModel): - pass - class CustomCollection(SerialisedModelCollection): pass - with pytest.raises(TypeError) as e: - SerialisedModel() - - if sys.version_info >= (3, 12): - assert str(e.value) == ( - "Can't instantiate abstract class SerialisedModel without an implementation " - "for abstract method 'ALLOWED_PROPERTIES'" - ) - else: - assert str(e.value) == ( - "Can't instantiate abstract class SerialisedModel with abstract method ALLOWED_PROPERTIES" - ) - - with pytest.raises(TypeError) as e: - Custom() - - if sys.version_info >= (3, 12): - assert str(e.value) == ( - "Can't instantiate abstract class Custom without an implementation " - "for abstract method 'ALLOWED_PROPERTIES'" - ) - else: - assert str(e.value) == "Can't instantiate abstract class Custom with abstract method ALLOWED_PROPERTIES" - with pytest.raises(TypeError) as e: SerialisedModelCollection() From 9705593862caa9934aab094b6e58d98be2d09c9f Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 20 Sep 2024 09:31:18 +0100 Subject: [PATCH 131/211] Rewrite `SerialisedModel` to use annotations Instead of an arbitrarily-named `set` floating around we can define fields directly on the class itself using annotations[1] syntax. The annotations syntax is not something which existed when we created this class. This also gives us somewhere to specify type information. This means we could start to do: - validtion - default values - type coercion *** 1. https://docs.python.org/3/howto/annotations.html --- notifications_utils/serialised_model.py | 8 ++++---- tests/test_serialised_model.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/notifications_utils/serialised_model.py b/notifications_utils/serialised_model.py index 27c368e91..3662e1c85 100644 --- a/notifications_utils/serialised_model.py +++ b/notifications_utils/serialised_model.py @@ -9,19 +9,19 @@ class SerialisedModel: that it can be interacted with like any other object. It is cleaner and safer than dealing with dictionaries directly because it guarantees that: - - all of the ALLOWED_PROPERTIES are present in the underlying + - all of the fields in its annotations are present in the underlying dictionary - any other abritrary properties of the underlying dictionary can’t be accessed If you are adding a new field to a model, you should ensure that all sources of the cache data are updated to return that new field, - then clear the cache, before adding that field to the - ALLOWED_PROPERTIES list. + then clear the cache, before adding that field to the class’s + annotations. """ def __init__(self, _dict): - for property in self.ALLOWED_PROPERTIES: + for property in getattr(self, "__annotations__", {}): setattr(self, property, _dict[property]) diff --git a/tests/test_serialised_model.py b/tests/test_serialised_model.py index fa84ddc40..8920199af 100644 --- a/tests/test_serialised_model.py +++ b/tests/test_serialised_model.py @@ -1,4 +1,5 @@ import sys +from typing import Any import pytest @@ -36,14 +37,14 @@ class CustomCollection(SerialisedModelCollection): def test_looks_up_from_dict(): class Custom(SerialisedModel): - ALLOWED_PROPERTIES = {"foo"} + foo: Any assert Custom({"foo": "bar"}).foo == "bar" def test_cant_override_custom_property_from_dict(): class Custom(SerialisedModel): - ALLOWED_PROPERTIES = {"foo"} + foo: Any @property def foo(self): @@ -66,12 +67,10 @@ def foo(self): ) def test_model_raises_for_unknown_attributes(json_response): class Custom(SerialisedModel): - ALLOWED_PROPERTIES = set() + pass model = Custom(json_response) - assert model.ALLOWED_PROPERTIES == set() - with pytest.raises(AttributeError) as e: model.foo # noqa @@ -80,7 +79,7 @@ class Custom(SerialisedModel): def test_model_raises_keyerror_if_item_missing_from_dict(): class Custom(SerialisedModel): - ALLOWED_PROPERTIES = {"foo"} + foo: Any with pytest.raises(KeyError) as e: Custom({}).foo # noqa @@ -97,7 +96,6 @@ class Custom(SerialisedModel): ) def test_model_doesnt_swallow_attribute_errors(json_response): class Custom(SerialisedModel): - ALLOWED_PROPERTIES = set() @property def foo(self): @@ -111,7 +109,9 @@ def foo(self): def test_dynamic_properties_are_introspectable(): class Custom(SerialisedModel): - ALLOWED_PROPERTIES = {"foo", "bar", "baz"} + foo: Any + bar: Any + baz: Any instance = Custom({"foo": "", "bar": "", "baz": ""}) @@ -130,7 +130,7 @@ class CustomCollection(SerialisedModelCollection): def test_serialised_model_collection_returns_models_from_list(): class Custom(SerialisedModel): - ALLOWED_PROPERTIES = {"x"} + x: Any class CustomCollection(SerialisedModelCollection): model = Custom From bd74a5c04c5c11c89d4bd31bc0392241100c71ed Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 20 Sep 2024 09:50:28 +0100 Subject: [PATCH 132/211] Add type coercion for primitive types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pretty straightforward, gives a guarantee that a property will be of a certain type. We treat datetimes as a special – and very useful – case because: - the native datetime type won’t do this conversion automatically - it avoids us passing-around datetime strings and datetimes in various states of timezone awareness --- notifications_utils/serialised_model.py | 19 ++++++++- tests/test_serialised_model.py | 51 ++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/notifications_utils/serialised_model.py b/notifications_utils/serialised_model.py index 3662e1c85..486b6f9c7 100644 --- a/notifications_utils/serialised_model.py +++ b/notifications_utils/serialised_model.py @@ -1,4 +1,8 @@ from abc import ABC, abstractmethod +from datetime import UTC, datetime +from typing import Any + +from notifications_utils.timezones import utc_string_to_aware_gmt_datetime class SerialisedModel: @@ -21,8 +25,19 @@ class SerialisedModel: """ def __init__(self, _dict): - for property in getattr(self, "__annotations__", {}): - setattr(self, property, _dict[property]) + for property, type_ in getattr(self, "__annotations__", {}).items(): + value = self.coerce_value_to_type(_dict[property], type_) + setattr(self, property, value) + + @staticmethod + def coerce_value_to_type(value, type_): + if type_ is Any or value is None: + return value + + if issubclass(type_, datetime): + return utc_string_to_aware_gmt_datetime(value).astimezone(UTC) + + return type_(value) class SerialisedModelCollection(ABC): diff --git a/tests/test_serialised_model.py b/tests/test_serialised_model.py index 8920199af..4281575eb 100644 --- a/tests/test_serialised_model.py +++ b/tests/test_serialised_model.py @@ -1,5 +1,7 @@ import sys +from datetime import UTC, datetime from typing import Any +from uuid import UUID import pytest @@ -115,7 +117,54 @@ class Custom(SerialisedModel): instance = Custom({"foo": "", "bar": "", "baz": ""}) - assert dir(instance)[-3:] == ["bar", "baz", "foo"] + for field in ("bar", "baz", "foo"): + assert field in dir(instance) + + +def test_none_values_are_not_coerced(): + class Custom(SerialisedModel): + foo: str + bar: int + + instance = Custom({"foo": None, "bar": None}) + + assert instance.foo is None + assert instance.bar is None + + +def test_types_are_coerced(): + class Custom(SerialisedModel): + id: UUID + year: str + version: int + rate: float + created_at: datetime + + instance = Custom( + { + "id": "bf777b2c-2bbd-487f-a09f-62ad46a9f92b", + "year": 2024, + "version": "9", + "rate": "1.234", + "created_at": "2024-03-02T01:00:00.000000Z", + } + ) + + assert instance.id == UUID("bf777b2c-2bbd-487f-a09f-62ad46a9f92b") + assert instance.year == "2024" + assert instance.version == 9 + assert instance.rate == 1.234 + assert instance.created_at == datetime(2024, 3, 2, 1, 0, tzinfo=UTC) + + +def test_raises_if_coercion_fails(): + class Custom(SerialisedModel): + version: int + + with pytest.raises(ValueError) as e: + Custom({"version": "twelvty"}) + + assert str(e.value) == "invalid literal for int() with base 10: 'twelvty'" def test_empty_serialised_model_collection(): From fcf14669904570a9d31946729c53c12af2b4af31 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 20 Sep 2024 09:56:50 +0100 Subject: [PATCH 133/211] Support inheritence of annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By default (in Python 3.10 and newer) annotations don’t inherit from one class to another. However we are mimicking inheritence by merging the `ALLOWED_PROPERTIES` sets on the various `Branding` subclasses. So let’s make it so it just works. It does this by looping over the current class’s parents (in method resolution order[1]) and merging each parent’s annotations with its child’s. *** 1. https://docs.python.org/3/howto/mro.html#python-2-3-mro --- notifications_utils/serialised_model.py | 7 ++++++- tests/test_serialised_model.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/notifications_utils/serialised_model.py b/notifications_utils/serialised_model.py index 486b6f9c7..0c6f780b5 100644 --- a/notifications_utils/serialised_model.py +++ b/notifications_utils/serialised_model.py @@ -24,8 +24,13 @@ class SerialisedModel: annotations. """ + def __new__(cls, *args, **kwargs): + for parent in cls.__mro__: + cls.__annotations__ = getattr(parent, "__annotations__", {}) | cls.__annotations__ + return super().__new__(cls) + def __init__(self, _dict): - for property, type_ in getattr(self, "__annotations__", {}).items(): + for property, type_ in self.__annotations__.items(): value = self.coerce_value_to_type(_dict[property], type_) setattr(self, property, value) diff --git a/tests/test_serialised_model.py b/tests/test_serialised_model.py index 4281575eb..99747d050 100644 --- a/tests/test_serialised_model.py +++ b/tests/test_serialised_model.py @@ -121,6 +121,24 @@ class Custom(SerialisedModel): assert field in dir(instance) +def test_attribute_inheritence(): + class Parent1(SerialisedModel): + foo: str + + class Parent2(SerialisedModel): + bar: str + + class Child(Parent1, Parent2): + __sort_attribute__ = "foo" + baz: str + + instance = Child({"foo": 1, "bar": 2, "baz": 3}) + + assert instance.foo == "1" + assert instance.bar == "2" + assert instance.baz == "3" + + def test_none_values_are_not_coerced(): class Custom(SerialisedModel): foo: str From e5cfe7992dc9195542ae8bbd5c2aae3aceb8ffca Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 20 Sep 2024 10:05:51 +0100 Subject: [PATCH 134/211] Major version bump --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96e4188fc..4514aa201 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 85.0.0 + +* Removes `SerialisedModel.ALLOWED_PROPERTIES` in favour of annotations syntax + ## 84.3.0 * Reverts 84.1.0 diff --git a/notifications_utils/version.py b/notifications_utils/version.py index e04ae45b3..00000fec4 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "84.3.0" # e8f0e1b3e2ed73b2e0962487a0292e38 +__version__ = "85.0.0" # cd0abf61c0669711a30b4f9c4dbb6f93 From 8bd1c4bec20a074c6d9c779634f9a1719b36e2af Mon Sep 17 00:00:00 2001 From: Richard Parke Date: Fri, 30 Aug 2024 16:45:59 +0100 Subject: [PATCH 135/211] Separate out parsing of possible numbers and validation of allowed numbers Previously, the init method on PhoneNumbers objects did both parsing and validation. This causes a number of small problems. Firstly, it makes api code difficult to read. There are instances where instansiation of a PhoneNumber object is being used to validate a number against a services permissions; this isn't clear from the code. This also makes the class much less flexible because it becomes tightly bound to a given services permissions. --- .../recipient_validation/phone_number.py | 68 ++++++++++--------- notifications_utils/recipients.py | 8 +-- notifications_utils/version.py | 2 +- .../recipient_validation/test_phone_number.py | 54 ++++++++++----- 4 files changed, 77 insertions(+), 55 deletions(-) diff --git a/notifications_utils/recipient_validation/phone_number.py b/notifications_utils/recipient_validation/phone_number.py index cf68d4349..e495e8328 100644 --- a/notifications_utils/recipient_validation/phone_number.py +++ b/notifications_utils/recipient_validation/phone_number.py @@ -174,37 +174,44 @@ def format_phone_number_human_readable(phone_number): class PhoneNumber: """ - A class that contains phone number validation. + A class that parses and performs validation checks on phonenumbers against service permissions - Supports mobile and landline numbers. When creating an object you must specify whether you are expecting - international phone numbers to be allowed or not. + Can be instantiated for all Phone Numbers other than premium numbers. Instansiation checks that the number + you are trying to send to is possible, validate checks the number against a services permissions passed to it + to ensure it can send. + + Examples: + number = PhoneNumber("07910777555") + number.validate(allow_international_number = False, allow_uk_landline = False) """ - def __init__(self, phone_number: str, *, allow_international: bool, allow_landline: bool = False) -> None: - self.raw_input = phone_number - self.allow_international = allow_international - self.allow_landline = allow_landline + def __init__(self, phone_number: str) -> None: try: - self.number = self.validate_phone_number(phone_number) + self.number = self.parse_phone_number(phone_number) except InvalidPhoneError: phone_number = self._thoroughly_normalise_number(phone_number) - self.number = self.validate_phone_number(phone_number) - self._raise_if_service_cannot_send_to_uk_landline_but_tries_to(phone_number) - self._raise_if_service_cannot_send_to_international_but_tries_to(phone_number) + self.number = self.parse_phone_number(phone_number) + self._phone_number = phone_number - def _raise_if_service_cannot_send_to_international_but_tries_to(self, phone_number): - number = self._try_parse_number(phone_number) - if not self.allow_international and str(number.country_code) != UK_PREFIX: + def _raise_if_service_cannot_send_to_international_but_tries_to(self, allow_international: bool = False): + number = self._try_parse_number(self._phone_number) + if not allow_international and str(number.country_code) != UK_PREFIX: raise InvalidPhoneError(code=InvalidPhoneError.Codes.NOT_A_UK_MOBILE) - def _raise_if_service_cannot_send_to_uk_landline_but_tries_to(self, phone_number): - phone_number = self._try_parse_number(phone_number) - if phone_number.country_code != 44: + def _raise_if_service_cannot_send_to_uk_landline_but_tries_to( + self, allow_uk_landline: bool = False, allow_international_number: bool = False + ): + phone_number = self._try_parse_number(self._phone_number) + if phone_number.country_code != int(UK_PREFIX): return is_landline = phonenumbers.number_type(phone_number) in LANDLINE_CODES - if not self.allow_landline and is_landline: + if not allow_uk_landline and is_landline: raise InvalidPhoneError(code=InvalidPhoneError.Codes.NOT_A_UK_MOBILE) + def validate(self, allow_international_number: bool = False, allow_uk_landline: bool = False) -> None: + self._raise_if_service_cannot_send_to_international_but_tries_to(allow_international=allow_international_number) + self._raise_if_service_cannot_send_to_uk_landline_but_tries_to(allow_uk_landline=allow_uk_landline) + @staticmethod def _try_parse_number(phone_number): try: @@ -213,22 +220,23 @@ def _try_parse_number(phone_number): except phonenumbers.NumberParseException as e: raise InvalidPhoneError(code=InvalidPhoneError.Codes.INVALID_NUMBER) from e + @staticmethod + def _raise_if_phone_number_is_empty(number: str) -> None: + if number == "" or number is None: + raise InvalidPhoneError(code=InvalidPhoneError.Codes.TOO_SHORT) + @staticmethod def _raise_if_phone_number_contains_invalid_characters(number: str) -> None: chars = set(number) if chars - {*ALL_WHITESPACE + "()-+" + "0123456789"}: raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNKNOWN_CHARACTER) - @staticmethod - def _raise_if_phone_number_is_empty(number: str) -> None: - if number == "": - raise InvalidPhoneError(code=InvalidPhoneError.Codes.TOO_SHORT) - - def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: + def parse_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: """ - Validate a phone number and return the PhoneNumber object + Parse a phone number and return the PhoneNumber object - Tries best effort validation, and has some extra logic to make the validation closer to our existing validation + Tries best effort parsing of a number (without any consideration for a services permissions), and has some extra + logic to make the validation closer to our existing validation including: * Being stricter with rogue alphanumeric characters. (eg don't allow https://en.wikipedia.org/wiki/Phoneword) @@ -236,12 +244,10 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: changes whether it is parsed as international or not. * Convert error codes to match existing Notify error codes """ - - self._raise_if_phone_number_is_empty(phone_number) - # notify's old validation code is stricter than phonenumbers in not allowing letters etc, so need to catch some # of those cases separately before we parse with the phonenumbers library self._raise_if_phone_number_contains_invalid_characters(phone_number) + self._raise_if_phone_number_is_empty(phone_number) number = self._try_parse_number(phone_number) @@ -249,9 +255,7 @@ def validate_phone_number(self, phone_number: str) -> phonenumbers.PhoneNumber: raise InvalidPhoneError(code=InvalidPhoneError.Codes.UNSUPPORTED_COUNTRY_CODE) if (reason := phonenumbers.is_possible_number_with_reason(number)) != phonenumbers.ValidationResult.IS_POSSIBLE: - if self.allow_international and ( - forced_international_number := self._validate_forced_international_number(phone_number) - ): + if forced_international_number := self._validate_forced_international_number(phone_number): number = forced_international_number else: raise InvalidPhoneError.from_phonenumbers_validation_result(reason) diff --git a/notifications_utils/recipients.py b/notifications_utils/recipients.py index 85449d088..90fd6c8b0 100644 --- a/notifications_utils/recipients.py +++ b/notifications_utils/recipients.py @@ -321,10 +321,10 @@ def _get_error_for_field(self, key, value): # noqa: C901 email_address.validate_email_address(value) if self.template_type == "sms": if self.allow_sms_to_uk_landline: - PhoneNumber( - value, - allow_international=self.allow_international_sms, - allow_landline=self.allow_sms_to_uk_landline, + number = PhoneNumber(value) + number.validate( + allow_international_number=self.allow_international_sms, + allow_uk_landline=self.allow_sms_to_uk_landline, ) else: phone_number.validate_phone_number( diff --git a/notifications_utils/version.py b/notifications_utils/version.py index e04ae45b3..da08f0f52 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "84.3.0" # e8f0e1b3e2ed73b2e0962487a0292e38 +__version__ = "85.0.0" # 3ddeb73a2a12440eb7213b918b644d08 diff --git a/tests/recipient_validation/test_phone_number.py b/tests/recipient_validation/test_phone_number.py index c0169e803..010051142 100644 --- a/tests/recipient_validation/test_phone_number.py +++ b/tests/recipient_validation/test_phone_number.py @@ -404,13 +404,13 @@ class TestPhoneNumberClass: def test_rejects_invalid_uk_mobile_phone_numbers(self, phone_number, error_message): # problem is `invalid_uk_mobile_phone_numbers` also includes valid uk landlines with pytest.raises(InvalidPhoneError): - PhoneNumber(phone_number, allow_international=False) + PhoneNumber(phone_number) # assert e.value.code == InvalidPhoneError.Codes.INVALID_NUMBER @pytest.mark.parametrize("phone_number", invalid_uk_landlines) def test_rejects_invalid_uk_landlines(self, phone_number): with pytest.raises(InvalidPhoneError) as e: - PhoneNumber(phone_number, allow_international=False, allow_landline=True) + PhoneNumber(phone_number) assert e.value.code == InvalidPhoneError.Codes.INVALID_NUMBER @pytest.mark.parametrize( @@ -425,29 +425,30 @@ def test_rejects_invalid_uk_landlines(self, phone_number): ) def test_rejects_invalid_international_phone_numbers(self, phone_number, error_message): with pytest.raises(InvalidPhoneError): - PhoneNumber(phone_number, allow_international=True) + PhoneNumber(phone_number) @pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) def test_allows_valid_uk_mobile_phone_numbers(self, phone_number): - assert PhoneNumber(phone_number, allow_international=False).is_uk_phone_number() is True + assert PhoneNumber(phone_number).is_uk_phone_number() is True @pytest.mark.parametrize("phone_number", valid_international_phone_numbers) def test_allows_valid_international_phone_numbers(self, phone_number): - assert PhoneNumber(phone_number, allow_international=True).is_uk_phone_number() is False + assert PhoneNumber(phone_number).is_uk_phone_number() is False @pytest.mark.parametrize("phone_number", valid_uk_landlines) def test_allows_valid_uk_landlines(self, phone_number): - assert PhoneNumber(phone_number, allow_international=True, allow_landline=True).is_uk_phone_number() is True + assert PhoneNumber(phone_number).is_uk_phone_number() is True @pytest.mark.parametrize("phone_number", valid_uk_landlines) def test_rejects_valid_uk_landlines_if_allow_landline_is_false(self, phone_number): with pytest.raises(InvalidPhoneError) as exc: - PhoneNumber(phone_number, allow_international=True, allow_landline=False) + number = PhoneNumber(phone_number) + number.validate(allow_international_number=True, allow_uk_landline=False) assert exc.value.code == InvalidPhoneError.Codes.NOT_A_UK_MOBILE @pytest.mark.parametrize("phone_number, expected_info", international_phone_info_fixtures) def test_get_international_phone_info(self, phone_number, expected_info): - assert PhoneNumber(phone_number, allow_international=True).get_international_phone_info() == expected_info + assert PhoneNumber(phone_number).get_international_phone_info() == expected_info @pytest.mark.parametrize( "number, expected", @@ -459,11 +460,11 @@ def test_get_international_phone_info(self, phone_number, expected_info): ], ) def test_should_use_numeric_sender(self, number, expected): - assert PhoneNumber(number, allow_international=True).should_use_numeric_sender() == expected + assert PhoneNumber(number).should_use_numeric_sender() == expected @pytest.mark.parametrize("phone_number", valid_uk_mobile_phone_numbers) def test_get_normalised_format_works_for_uk_mobiles(self, phone_number): - assert PhoneNumber(phone_number, allow_international=True).get_normalised_format() == "447723456789" + assert PhoneNumber(phone_number).get_normalised_format() == "447723456789" @pytest.mark.parametrize( "phone_number, expected_formatted", @@ -477,7 +478,7 @@ def test_get_normalised_format_works_for_uk_mobiles(self, phone_number): ], ) def test_get_normalised_format_works_for_international_numbers(self, phone_number, expected_formatted): - assert PhoneNumber(phone_number, allow_international=True).get_normalised_format() == expected_formatted + assert str(PhoneNumber(phone_number)) == expected_formatted @pytest.mark.parametrize( "phone_number, expected_formatted", @@ -495,7 +496,7 @@ def test_get_normalised_format_works_for_international_numbers(self, phone_numbe ], ) def test_get_human_readable_format(self, phone_number, expected_formatted): - assert PhoneNumber(phone_number, allow_international=True).get_human_readable_format() == expected_formatted + assert PhoneNumber(phone_number).get_human_readable_format() == expected_formatted # TODO: when we've removed the old style validation, we can just roll these in to our regular test fixtures # eg valid_uk_landline, invalid_uk_mobile_number, valid_international_number @@ -522,7 +523,7 @@ def test_get_human_readable_format(self, phone_number, expected_formatted): ], ) def test_validate_normalised_succeeds(self, phone_number, expected_normalised_number): - normalised_number = PhoneNumber(phone_number, allow_international=True, allow_landline=True) + normalised_number = PhoneNumber(phone_number) assert str(normalised_number) == expected_normalised_number # TODO: decide if all these tests are useful to have. @@ -552,7 +553,7 @@ def test_validate_normalised_succeeds(self, phone_number, expected_normalised_nu ) def test_validate_normalised_fails(self, phone_number, expected_error_code): with pytest.raises(InvalidPhoneError) as exc: - PhoneNumber(phone_number, allow_international=True) + PhoneNumber(phone_number) assert exc.value.code == expected_error_code @pytest.mark.parametrize( @@ -560,7 +561,7 @@ def test_validate_normalised_fails(self, phone_number, expected_error_code): [("07700900010", "447700900010"), ("447700900020", "447700900020"), ("+447700900030", "447700900030")], ) def test_tv_number_passes(self, phone_number, expected_valid_number): - number = PhoneNumber(phone_number, allow_international=True) + number = PhoneNumber(phone_number) assert expected_valid_number == str(number) @pytest.mark.parametrize( @@ -575,16 +576,32 @@ def test_tv_number_passes(self, phone_number, expected_valid_number): ) def test_international_does_not_normalise_to_uk_number(self, phone_number, expected_error_code): with pytest.raises(InvalidPhoneError) as exc: - PhoneNumber(phone_number, allow_international=False, allow_landline=True) + number = PhoneNumber(phone_number) + number.validate(allow_international_number=False, allow_uk_landline=True) assert exc.value.code == expected_error_code + @pytest.mark.parametrize( + "phone_number", valid_uk_mobile_phone_numbers + valid_international_phone_numbers + valid_channel_island_numbers + ) + def test_all_valid_numbers_parse_regardless_of_service_permissions(self, phone_number): + """ + The PhoneNumber class should parse all numbers on instantiation regardless of permissions if they're + a possible phone number. Checking whether a user or service can send that number should only be handled + by the validate_phone_number method. + """ + try: + PhoneNumber(phone_number) + except InvalidPhoneError: + pytest.fail("Unexpected InvalidPhoneError") + # We discovered a bug with the phone_numbers library causing some valid JE numbers # to evaluate as invalid. Realiably sending to Crown Dependencies is very important # this test serves to alert us if a known failing edge case arises again. @pytest.mark.parametrize("phone_number", valid_channel_island_numbers) def test_channel_island_numbers_are_valid(self, phone_number): try: - PhoneNumber(phone_number, allow_international=True) + number = PhoneNumber(phone_number) + number.validate(allow_international_number=True, allow_uk_landline=False) except InvalidPhoneError: pytest.fail("Unexpected InvalidPhoneError") @@ -593,5 +610,6 @@ def test_empty_phone_number_is_rejected_with_correct_v2_error_message(): phone_number = "" error_message = InvalidPhoneError(code=InvalidPhoneError.Codes.TOO_SHORT) with pytest.raises(InvalidPhoneError) as e: - PhoneNumber(phone_number=phone_number, allow_international=True) + number = PhoneNumber(phone_number=phone_number) + number.validate(allow_international_number=True, allow_uk_landline=False) assert str(error_message) == str(e.value) From c205017152448763612fc90dbf9a523c204279b4 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 27 Sep 2024 12:28:23 +0100 Subject: [PATCH 136/211] Rename test for clarity --- tests/test_asset_fingerprinter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_asset_fingerprinter.py b/tests/test_asset_fingerprinter.py index 96b367370..f7a783ea3 100644 --- a/tests/test_asset_fingerprinter.py +++ b/tests/test_asset_fingerprinter.py @@ -55,7 +55,7 @@ def test_hashes_are_different_for_different_files(self, mocker): js_hash = asset_fingerprinter.get_asset_fingerprint("application.js") assert js_hash != css_hash - def test_hash_gets_cached(self, mocker): + def test_gets_fingerprint_from_cache(self, mocker): get_file_content_mock = mocker.patch.object(AssetFingerprinter, "get_asset_file_contents") get_file_content_mock.return_value = b""" body { From 924bd925637d8e9572cd29ce40cbb4841b1d507c Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Wed, 2 Oct 2024 14:41:13 +0100 Subject: [PATCH 137/211] add EventletTimeoutMiddleware including eventlet detection that will omit full import if not using eventlet. this is needed because gunicorn doesn't do this out-of-the-box for the eventlet worker. instead the `timeout` configuration variable controls when the whole worker process will be killed if it hasn't produced *any* output for a certain time - it doesn't work on a per-request level. --- notifications_utils/eventlet.py | 54 +++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 notifications_utils/eventlet.py diff --git a/notifications_utils/eventlet.py b/notifications_utils/eventlet.py new file mode 100644 index 000000000..b0024a764 --- /dev/null +++ b/notifications_utils/eventlet.py @@ -0,0 +1,54 @@ +import sys +from collections.abc import Callable + + +# eventlet's own Timeout class inherits from BaseException instead of +# Exception, which makes more likely that an attempted catch-all +# handler will miss it. +class EventletTimeout(Exception): + pass + + +# eventlet detection cribbed from +# https://github.com/celery/kombu/blob/74779a8078ab318a016ca107249e59f8c8063ef9/kombu/utils/compat.py#L38 +using_eventlet = False +if "eventlet" in sys.modules: + import socket + + try: + from eventlet.patcher import is_monkey_patched + except ImportError: + pass + else: + if is_monkey_patched(socket): + using_eventlet = True + +if using_eventlet: + from eventlet.timeout import Timeout + + class EventletTimeoutMiddleware: + """ + A WSGI middleware that will raise `exception` after `timeout_seconds` of request + processing, *but only when* the next I/O context switch occurs. + """ + + _app: Callable + _timeout_seconds: float + _exception: BaseException + + def __init__( + self, + app: Callable, + timeout_seconds: float = 30, + exception: BaseException = EventletTimeout, + ): + self._app = app + self._timeout_seconds = timeout_seconds + self._exception = exception + + def __call__(self, *args, **kwargs): + with Timeout(self._timeout_seconds, exception=self._exception): + return self._app(*args, **kwargs) + +else: + EventletTimeoutMiddleware = None From 024987d0fee2dce73552306b8c21b9b06c9557d4 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Wed, 2 Oct 2024 14:58:50 +0100 Subject: [PATCH 138/211] Bump version to 86.1.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 057c4d7b0..611978372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 86.1.0 + +* Add `EventletTimeoutMiddleware` + # 86.0.0 * BREAKING CHANGE: The `Phonenumber` class now accepts a flag `allow_landline`, which defaults to False. This changes the previous default behaviour, allowing landlines. diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 3d2ec0633..be63cb75b 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "86.0.0" # 0364f94071752009eb402e04c5b79635 +__version__ = "86.1.0" # 12f54a9765ef57d2945475f8dda35ab0 From c5a9a43eae2c6639c64b3cf72da7d6c9769df6e3 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Mon, 16 Sep 2024 13:11:50 +0100 Subject: [PATCH 139/211] Revert "Revert "ZendeskClient: use a persistent requests session"" This reverts commit 259c8e904858e0adeec7ab42c690feb4e4d9822a. --- .../clients/zendesk/zendesk_client.py | 20 +++++++++++-------- tests/clients/zendesk/test_zendesk_client.py | 12 +++-------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/notifications_utils/clients/zendesk/zendesk_client.py b/notifications_utils/clients/zendesk/zendesk_client.py index 12cf0fd8c..6d31b6bbe 100644 --- a/notifications_utils/clients/zendesk/zendesk_client.py +++ b/notifications_utils/clients/zendesk/zendesk_client.py @@ -49,6 +49,12 @@ class NotifySupportTicketComment: class ZendeskClient: + """ + A client for the Zendesk API + + This class is not thread-safe. + """ + # the account used to authenticate with. If no requester is provided, the ticket will come from this account. NOTIFY_ZENDESK_EMAIL = "zd-api-notify@digital.cabinet-office.gov.uk" @@ -56,14 +62,12 @@ class ZendeskClient: ZENDESK_UPDATE_TICKET_URL = "https://govuk.zendesk.com/api/v2/tickets/{ticket_id}" ZENDESK_UPLOAD_FILE_URL = "https://govuk.zendesk.com/api/v2/uploads.json" - def __init__(self): - self.api_key = None - - def init_app(self, app, *args, **kwargs): - self.api_key = app.config.get("ZENDESK_API_KEY") + def __init__(self, api_key): + self.api_key = api_key + self.requests_session = requests.Session() def send_ticket_to_zendesk(self, ticket): - response = requests.post( + response = self.requests_session.post( self.ZENDESK_TICKET_URL, json=ticket.request_data, auth=(f"{self.NOTIFY_ZENDESK_EMAIL}/token", self.api_key) ) @@ -92,7 +96,7 @@ def _upload_attachment(self, attachment: NotifySupportTicketAttachment): upload_url = self.ZENDESK_UPLOAD_FILE_URL + "?" + urlencode(query_params) - response = requests.post( + response = self.requests_session.post( upload_url, headers={"Content-Type": attachment.content_type}, data=attachment.filedata, @@ -138,7 +142,7 @@ def update_ticket( data["ticket"]["status"] = status.value update_url = self.ZENDESK_UPDATE_TICKET_URL.format(ticket_id=ticket_id) - response = requests.put( + response = self.requests_session.put( update_url, json=data, auth=(f"{self.NOTIFY_ZENDESK_EMAIL}/token", self.api_key), diff --git a/tests/clients/zendesk/test_zendesk_client.py b/tests/clients/zendesk/test_zendesk_client.py index a30b2317e..bd121c9bf 100644 --- a/tests/clients/zendesk/test_zendesk_client.py +++ b/tests/clients/zendesk/test_zendesk_client.py @@ -17,14 +17,8 @@ @pytest.fixture(scope="function") -def zendesk_client(app): - client = ZendeskClient() - - app.config["ZENDESK_API_KEY"] = "testkey" - - client.init_app(app) - - return client +def zendesk_client(): + return ZendeskClient(api_key="testkey") def test_zendesk_client_send_ticket_to_zendesk(zendesk_client, app, rmock, caplog): @@ -63,7 +57,7 @@ def test_zendesk_client_send_ticket_to_zendesk_error(zendesk_client, app, rmock, assert "Zendesk create ticket request failed with 401 '{'foo': 'bar'}'" in caplog.messages -def test_zendesk_client_send_ticket_to_zendesk_with_user_suspended_error(zendesk_client, rmock, caplog): +def test_zendesk_client_send_ticket_to_zendesk_with_user_suspended_error(zendesk_client, app, rmock, caplog): rmock.request( "POST", ZendeskClient.ZENDESK_TICKET_URL, From fb7776c9dc7b2bb6882b51e2e73bfcdb9900e42a Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Mon, 16 Sep 2024 13:12:05 +0100 Subject: [PATCH 140/211] Revert "Revert "AntivirusClient: use a persistent requests session"" This reverts commit e7f981432de90f8c18de20e4eb332c5161176a48. --- .../clients/antivirus/antivirus_client.py | 13 ++++++++----- tests/clients/antivirus/test_antivirus_client.py | 8 ++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/notifications_utils/clients/antivirus/antivirus_client.py b/notifications_utils/clients/antivirus/antivirus_client.py index f723db310..74a165af3 100644 --- a/notifications_utils/clients/antivirus/antivirus_client.py +++ b/notifications_utils/clients/antivirus/antivirus_client.py @@ -21,13 +21,16 @@ def from_exception(cls, e): class AntivirusClient: + """ + A client for the antivirus API + + This class is not thread-safe. + """ + def __init__(self, api_host=None, auth_token=None): self.api_host = api_host self.auth_token = auth_token - - def init_app(self, app): - self.api_host = app.config["ANTIVIRUS_API_HOST"] - self.auth_token = app.config["ANTIVIRUS_API_KEY"] + self.requests_session = requests.Session() def scan(self, document_stream): headers = {"Authorization": f"Bearer {self.auth_token}"} @@ -35,7 +38,7 @@ def scan(self, document_stream): headers.update(request.get_onwards_request_headers()) try: - response = requests.post( + response = self.requests_session.post( f"{self.api_host}/scan", headers=headers, files={"document": document_stream}, diff --git a/tests/clients/antivirus/test_antivirus_client.py b/tests/clients/antivirus/test_antivirus_client.py index 173ac0152..f5bab891a 100644 --- a/tests/clients/antivirus/test_antivirus_client.py +++ b/tests/clients/antivirus/test_antivirus_client.py @@ -12,10 +12,10 @@ @pytest.fixture(scope="function") def app_antivirus_client(app, mocker): - client = AntivirusClient() - app.config["ANTIVIRUS_API_HOST"] = "https://antivirus" - app.config["ANTIVIRUS_API_KEY"] = "test-antivirus-key" - client.init_app(app) + client = AntivirusClient( + api_host="https://antivirus", + auth_token="test-antivirus-key", + ) return app, client From 10234acaed7b6a90006c915861a7ca5223a660a9 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Mon, 16 Sep 2024 13:15:47 +0100 Subject: [PATCH 141/211] Bump version to 87.0.0 --- CHANGELOG.md | 4 ++++ notifications_utils/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bdf48f92..b6e96516f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG +## 87.0.0 + +* Reintroduce changes to `AntivirusClient` and `ZendeskClient` from 83.0.0 + ## 86.2.0 * Adds `asset_fingerprinter.AssetFingerprinter` to replace the versions duplicated across our frontend apps diff --git a/notifications_utils/version.py b/notifications_utils/version.py index 912d7d048..ab63a0c67 100644 --- a/notifications_utils/version.py +++ b/notifications_utils/version.py @@ -5,4 +5,4 @@ # - `make version-minor` for new features # - `make version-patch` for bug fixes -__version__ = "86.2.0" # 37dac8b3e2812ede962df582c8fe48ec +__version__ = "87.0.0" # 33dc08fb56c391a4737819a401da58ad From 8abff85975ef30b2d3b4cd6237e4dd06363336f1 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Fri, 25 Oct 2024 12:00:42 +0100 Subject: [PATCH 142/211] Remove `EmailPreviewTemplate` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is moved into the admin app as of: https://github.com/alphagov/notifications-admin/pull/5266/commits/e04175b85a7aecdff5b1c7f65cbfcc1631bc1b9f It’s not used anywhere else so we can safely remove it from this repo. --- notifications_utils/template.py | 48 --------- tests/test_recipient_csv.py | 4 +- tests/test_template_types.py | 172 -------------------------------- 3 files changed, 2 insertions(+), 222 deletions(-) diff --git a/notifications_utils/template.py b/notifications_utils/template.py index a2de68b48..37b50657c 100644 --- a/notifications_utils/template.py +++ b/notifications_utils/template.py @@ -562,54 +562,6 @@ def __str__(self): ) -class EmailPreviewTemplate(BaseEmailTemplate): - jinja_template = template_env.get_template("email_preview_template.jinja2") - - def __init__( - self, - template, - values=None, - from_name=None, - reply_to=None, - show_recipient=True, - redact_missing_personalisation=False, - **kwargs, - ): - super().__init__(template, values, redact_missing_personalisation=redact_missing_personalisation, **kwargs) - self.from_name = from_name - self.reply_to = reply_to - self.show_recipient = show_recipient - - def __str__(self): - return Markup( - self.jinja_template.render( - { - "body": self.html_body, - "subject": self.subject, - "from_name": escape_html(self.from_name), - "reply_to": self.reply_to, - "recipient": Field("((email address))", self.values, with_brackets=False), - "show_recipient": self.show_recipient, - } - ) - ) - - @property - def subject(self): - return ( - Take( - Field( - self._subject, - self.values, - html="escape", - redact_missing_personalisation=self.redact_missing_personalisation, - ) - ) - .then(do_nice_typography) - .then(normalise_whitespace) - ) - - class BaseLetterTemplate(SubjectMixin, Template): template_type = "letter" max_page_count = LETTER_MAX_PAGE_COUNT diff --git a/tests/test_recipient_csv.py b/tests/test_recipient_csv.py index e4e4ca3dc..1a0c23794 100644 --- a/tests/test_recipient_csv.py +++ b/tests/test_recipient_csv.py @@ -18,7 +18,7 @@ first_column_headings, ) from notifications_utils.template import ( - EmailPreviewTemplate, + HTMLEmailTemplate, LetterPreviewTemplate, SMSMessageTemplate, ) @@ -26,7 +26,7 @@ def _sample_template(template_type, content="foo"): return { - "email": EmailPreviewTemplate({"content": content, "subject": "bar", "template_type": "email"}), + "email": HTMLEmailTemplate({"content": content, "subject": "bar", "template_type": "email"}), "sms": SMSMessageTemplate({"content": content, "template_type": "sms"}), "letter": LetterPreviewTemplate( {"content": content, "subject": "bar", "template_type": "letter"}, diff --git a/tests/test_template_types.py b/tests/test_template_types.py index e81c9702b..0f917470b 100644 --- a/tests/test_template_types.py +++ b/tests/test_template_types.py @@ -14,7 +14,6 @@ BaseEmailTemplate, BaseLetterTemplate, BaseSMSTemplate, - EmailPreviewTemplate, HTMLEmailTemplate, LetterPreviewTemplate, LetterPrintTemplate, @@ -356,7 +355,6 @@ def test_markdown_in_templates( "template_class, template_type, extra_attributes", [ (HTMLEmailTemplate, "email", 'style="word-wrap: break-word; color: #1D70B8;"'), - (EmailPreviewTemplate, "email", 'style="word-wrap: break-word; color: #1D70B8;"'), (SMSPreviewTemplate, "sms", 'class="govuk-link govuk-link--no-visited-state"'), pytest.param(SMSBodyPreviewTemplate, "sms", 'style="word-wrap: break-word;', marks=pytest.mark.xfail), ], @@ -940,7 +938,6 @@ def test_sms_templates_have_no_subject(template_class, template_json): def test_subject_line_gets_applied_to_correct_template_types(): for cls in [ - EmailPreviewTemplate, HTMLEmailTemplate, PlainTextEmailTemplate, LetterPreviewTemplate, @@ -957,7 +954,6 @@ def test_subject_line_gets_applied_to_correct_template_types(): @pytest.mark.parametrize( "template_class, template_type, extra_args", ( - (EmailPreviewTemplate, "email", {}), (HTMLEmailTemplate, "email", {}), (PlainTextEmailTemplate, "email", {}), (LetterPreviewTemplate, "letter", {}), @@ -974,7 +970,6 @@ def test_subject_line_gets_replaced(template_class, template_type, extra_args): @pytest.mark.parametrize( "template_class, template_type, extra_args", ( - (EmailPreviewTemplate, "email", {}), (HTMLEmailTemplate, "email", {}), (PlainTextEmailTemplate, "email", {}), (LetterPreviewTemplate, "letter", {}), @@ -1237,16 +1232,6 @@ def test_is_message_empty_email_and_letter_templates_tries_not_to_count_chars( mock.call("content", {}, html="escape", markdown_lists=True), ], ), - ( - EmailPreviewTemplate, - "email", - {}, - [ - mock.call("content", {}, html="escape", markdown_lists=True, redact_missing_personalisation=False), - mock.call("subject", {}, html="escape", redact_missing_personalisation=False), - mock.call("((email address))", {}, with_brackets=False), - ], - ), ( SMSMessageTemplate, "sms", @@ -1289,16 +1274,6 @@ def test_is_message_empty_email_and_letter_templates_tries_not_to_count_chars( mock.call("www.gov.uk", {}, html="escape", redact_missing_personalisation=False), ], ), - ( - EmailPreviewTemplate, - "email", - {"redact_missing_personalisation": True}, - [ - mock.call("content", {}, html="escape", markdown_lists=True, redact_missing_personalisation=True), - mock.call("subject", {}, html="escape", redact_missing_personalisation=True), - mock.call("((email address))", {}, with_brackets=False), - ], - ), ( SMSPreviewTemplate, "sms", @@ -1387,21 +1362,6 @@ def test_templates_handle_html_and_redacting( mock.call(Markup("subject")), ], ), - ( - EmailPreviewTemplate, - "email", - {}, - [ - mock.call( - '

    ' - "content" - "

    " - ), - mock.call(Markup("subject")), - mock.call(Markup("subject")), - mock.call(Markup("subject")), - ], - ), ( SMSMessageTemplate, "sms", @@ -1486,19 +1446,6 @@ def test_templates_remove_whitespace_before_punctuation( mock.call(Markup("subject")), ], ), - ( - EmailPreviewTemplate, - "email", - {}, - [ - mock.call( - '

    ' - "content" - "

    " - ), - mock.call(Markup("subject")), - ], - ), (SMSMessageTemplate, "sms", {}, []), (SMSPreviewTemplate, "sms", {}, []), (SMSBodyPreviewTemplate, "sms", {}, []), @@ -1548,7 +1495,6 @@ def test_templates_make_quotes_smart_and_dashes_en( ( HTMLEmailTemplate, PlainTextEmailTemplate, - EmailPreviewTemplate, ), ) def test_no_smart_quotes_in_email_addresses(template_class, content): @@ -1613,12 +1559,6 @@ def test_smart_quotes_removed_from_long_template_in_under_a_second(): ), ["subject", "content"], ), - ( - EmailPreviewTemplate( - {"content": "((content))", "subject": "((subject))", "template_type": "email"}, - ), - ["subject", "content"], - ), ( LetterPreviewTemplate( {"content": "((content))", "subject": "((subject))", "template_type": "letter"}, @@ -1647,62 +1587,6 @@ def test_html_template_can_inject_personalisation_with_special_characters(): ) -@pytest.mark.parametrize( - "extra_args", - [ - {"from_name": "Example service"}, - pytest.param({}, marks=pytest.mark.xfail), - ], -) -def test_email_preview_shows_from_name(extra_args): - template = EmailPreviewTemplate( - {"content": "content", "subject": "subject", "template_type": "email"}, **extra_args - ) - assert 'From' in str(template) - assert "Example service" in str(template) - - -def test_email_preview_escapes_html_in_from_name(): - template = EmailPreviewTemplate( - {"content": "content", "subject": "subject", "template_type": "email"}, from_name='' - ) - assert "