upstream alphagov updates

.git-blame-ignore-revs

@@ -1,2 +1,10 @@
 # bulk change for black formatting changes
 0f3f4b3e6aa5a199a1b5e79742aa776ce8d8bf7e
+ce619c18021e1a219cac2da4e3b0470ddfc08508
+
+# adding more linters to ruff
+26c86b6a8814b0f5948d9f1ea271d67141dbf50e
+38d37f09cb9472e60888bd5117f4e3b89c628b9f
+a8dc6c00113091ed735fa832881ff696fb1f632f
+8016ce3a6b16f3d30ed2ff6731fd20851c2b4803
+71483b5fb0251dd9e99f6cd45060df3c934cd7c1

.gitignore

@@ -43,6 +43,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .pytest_cache/
+.testmondata*
 
 # Translations
 *.mo

.pre-commit-config.yaml

@@ -0,0 +1 @@
+notifications_utils/version_tools/.pre-commit-config.yaml

CHANGELOG.md

@@ -1,5 +1,299 @@
 # CHANGELOG
 
+## 93.0.0
+
+* BREAKING CHANGE: logging: all contents of `logging/__init__.py` have been moved to `logging/flask.py` because they all assume a flask(-like) environment and this way we don't implicitly import all of flask etc. every time anything under `logging` is imported.
+* Adds `request_cpu_time` to `app.request` logs when available.
+* Adds ability to track eventlet's switching of greenlets and annotate `app.request` logs with useful metrics when the flask config parameter `NOTIFY_EVENTLET_STATS` is set and the newly provided function `account_greenlet_times` is installed as a greenlet trace hook.
+
+## 92.1.1
+
+* Bump minimum version of `jinja2` to 3.1.5
+
+## 92.1.0
+
+* RequestCache: add CacheResultWrapper to allow dynamic cache decisions
+
+## 92.0.2
+
+* Downgrade minimum version of `requests` to 2.32.3
+
+## 92.0.1
+
+* Bumps core dependencies to latest versions
+
+## 92.0.0
+
+* Restores old validation code so later utils changes can be added to api, admin etc.
+
+## 91.1.2
+
+* Adds rule N804 (invalid-first-argument-name-for-class-method) to linter config
+
+## 91.1.1
+
+* Remove hangul filler character from rendered emails
+
+## 91.1.0
+
+* bump all libs in requirements_for_test_common.in
+
+## 91.0.4
+
+* Don’t copy config when bumping utils version
+
+## 91.0.3
+
+* Fix validating supported international country codes for phone numbers without a leading "+" that look a bit like UK numbers
+
+## 91.0.2
+
+* Bumps requests to newer version
+
+## 91.0.1
+
+* Adds public utility method to check if a phonenumber is international (with correct handling for OFCOM TV numbers)
+* Updates PhoneNumber.get_international_phone_info to return the correct info for OFCOM TV numbers
+
+## 91.0.0
+
+* BREAKING CHANGE: All standalone functions for validating, normalising and formatting phone numbers has been removed from notifications_utils/recipient_validation/phone_number.py, and are replaced and encapsualated entirely with a single `PhoneNumber` class. All code that relies on validation, normalisation or fomratting of a phone number must be re-written to use instances of `PhoneNumber` instead.
+
+## 90.0.0
+
+* Allow leading empty columns in CSV files
+* Change second argument of `formatters.strip_all_whitespace` (this argument is not used in other apps)
+
+## 89.2.0
+
+* Use github API rather than fetching from their CDN in version_tools
+
+## 89.1.0
+
+* Change version_tools to a package that uses importlib to grab common reqs/config, rather than fetching common files from github. This repo's usage of those common files are now symlinks to the sources of truth found within notification_utils/version_tools/
+
+## 89.0.1
+
+* Raise an exception if we cant fetch remote github files in version_tools.py
+
+## 89.0.0
+
+* `requirements_for_test_common.txt` is now `requirements_for_test_common.in`. Apps should freeze this into a local requirements file for fully reproducible dependencies
+
+## 88.1.1
+
+* Bug fix: applies minor version bump of PhoneNumbers to v8.13.48 to apply a fix to the UG metadata which was causing validation to fail in error
+
+## 88.1.0
+
+* logging: slightly redesign filters behaviour to allow log-call-supplied request_id, span_id, user_id, service_id to be used if they are not available by normal means. Supply these manually for the streaming-response-closed log message.
+
+## 88.0.1
+
+* logging: don't use current_app in _log_response_closed
+
+## 88.0.0
+
+* Removes `template.EmailPreviewTemplate` (only used in admin app)
+
+## 87.1.0
+
+* logging: don't calculate_content_length() for after_request logs of streaming responses - this caused attempted streaming responses to be eagerly consumed, negating the point of streaming responses. Instead emit the after_request log when status code and header is returned and a separate log message when a streaming response is closed.
+
+## 87.0.0
+
+* Reintroduce changes to `AntivirusClient` and `ZendeskClient` from 83.0.0
+
+## 86.2.0
+
+* Adds `asset_fingerprinter.AssetFingerprinter` to replace the versions duplicated across our frontend apps
+
+## 86.1.0
+
+* Add `EventletTimeoutMiddleware`
+
+# 86.0.0
+
+* BREAKING CHANGE: The `Phonenumber` class now accepts a flag `allow_landline`, which defaults to False. This changes the previous default behaviour, allowing landlines.
+
+## 85.0.0
+
+* Removes `SerialisedModel.ALLOWED_PROPERTIES` in favour of annotations syntax
+
+## 84.3.0
+
+* Reverts 84.1.0
+
+## 84.2.0
+
+* The Zendesk client takes a new optional argument, `user_created_at` which populates a new field on the Notify Zendesk form if provided.
+
+## 84.1.1
+
+* Remove GIR 0AA from valid postcodes
+
+## 84.1.0
+
+* Bump versions of common test dependencies (run `make bootstrap` to copy these into your app)
+
+## 84.0.0
+
+* `AntivirusClient` and `ZendeskClient` have returned to their behaviour as of 82.x.x to allow the 83.0.1 fix to go out to apps without the required changes.
+
+# 83.0.1
+
+* Updates phone_numbers to 8.13.45 to apply a fix to the metadata for phone numbers that was discovered causing a subset of valid Jersey numbers to be incorrectly invalidated
+
+## 83.0.0
+
+* `AntivirusClient` and `ZendeskClient` are no longer thread-safe because they now use persistent requests sessions. Thread-local instances should be used in place of any global instances in any situations where threading is liable to be used
+* `AntivirusClient` and `ZendeskClient` have had their `init_app(...)` methods removed as this is an awkward pattern to use for initializing thread-local instances. It is recommended to use `LazyLocalGetter` to construct new instances on-demand, passing configuration parameters via the constructor arguments in a `factory` function.
+
+## 82.7.0
+
+* Add `expected_type` mechanism to `LazyLocalGetter`
+
+# 82.6.1
+
+* Adds validation check to `PhoneNumber` so that it returns the expected error message `TOO_SHORT` if an empty string is passed. This has caused issues with users of the v2 API getting inconsistent error messages
+
+## 82.6.0
+
+* Add `LazyLocalGetter` class for lazily-initialized context-local resources
+
+## 82.5.0
+
+* Add support for validation of UK landlines for services with sms_to_uk_landlines enabled using CSV flow
+
+## 82.4.0
+
+* Add support for sending SMS to more international numbers. Sending to Eritrea, Wallis and Futuna, Niue, Kiribati, and Tokelau is now supported.
+
+## 82.3.0
+* Extends the validation logic for the `PhoneNumber` class to disallow premium rate numbers
+
+## 82.2.1
+
+* Add fix to recipient_validation/phone_number.py to raise correct error if a service tries to send to an international number without that permission
+
+## 82.2.0
+* Add `unsubscribe_link` argument to email templates
+
+## 82.1.2
+* Write updated version number to `requirements.txt` if no `requirements.in` file found
+
+## 82.1.1
+*  Fix the way we log the request_size. Accessing the data at this point can trigger a validation error early and cause a 500 error
+
+## 82.1.0
+*  Adds new logging fields to request logging. Namely environment_name, request_size and response_size
+
+## 82.0.0
+
+* Change `PostalAddress` to add `has_no_fixed_abode_address` method. No fixed abode addresses are now considered invalid.
+
+## 81.1.1
+*  Adds condition to validation to allow TV Numbers (https://www.ofcom.org.uk/phones-and-broadband/phone-numbers/numbers-for-drama/) for UK mobiles
+
+## 81.1.0
+* introduce new validation class - `PhoneNumber`, that we will use for services that want to send sms
+to landline (and in the future this new code can be extended for all phone number validation)
+* in this new class, we use `phonenumbers` library for validating phone numbers, instead of our custom valdiation code
+
+
+## 81.0.0
+
+* BREAKING CHANGE: The constructor for `notification_utils.recipient_validation.errors.InvalidPhoneError`
+  - When raising InvalidPhoneError, instead of supplying a custom message, you must create an error by supplying a code from the `InvalidPhoneError.Codes` enum
+* `InvalidPhoneError.code` will contain this machine-readable code for an exception if you need to examine it later
+* `InvalidPhoneError.get_legacy_v2_api_error_message` returns a historical error message for use on the public v2 api
+
+## 80.0.1
+
+* Reduces minimum required Gunicorn version for compatibility
+
+## 80.0.0
+
+* Copies additional config files from utils into repos
+* Renames `version_tools.copy_pyproject_yaml` to `version_tools.copy_config`
+
+## 79.0.1
+
+* Update the `send_ticket_to_zendesk` method of the ZendeskClient to return the ID of the ticket that was created.
+
+## 79.0.0
+
+* Switches on Pyupgrade and a bunch of other more opinionated linting rules
+
+## 78.2.0
+
+* Bumped minimum versions of select subdependencies
+
+## 78.1.0
+
+* Restrict postcodes to valid UK postcode zones
+
+## 78.0.0
+
+* BREAKING CHANGE: recipient validation code has all been moved into separate files in a shared folder. Functionality is unchanged.
+  - Email address validation can be found in `notifications_utils.recipient_validation.email_address`
+  - Phone number validation can be found in `notifications_utils.recipient_validation.phone_number`
+  - Postal address validation can be found in `notifications_utils.recipient_validation.postal_address`
+* BREAKING CHANGE: InvalidPhoneError and InvalidAddressError no longer extend InvalidEmailError.
+  - if you wish to handle all recipient validation errors, please use `notifications_utils.recipient_validation.errors.InvalidRecipientError`
+
+## 77.2.1
+
+* Change redis delete behaviour to error, rather than end up with stale data, if Redis is unavailable.
+
+## 77.2.0
+
+* `NotifyTask`: include pid and other structured fields in completion log messages
+
+## 77.1.1
+
+* Fix how `version_tools.copy_pyproject_toml` discovers the current version of notifications-utils
+
+## 77.1.0
+
+* Add `version_tools.copy_pyproject_toml` to share linter config between apps
+
+## 77.0.0
+
+* Breaking change to the Zendesk Client. The `ticket_categories` argument has been replaced with a `notify_task_type` argument, and it now populates
+  a different Zendesk form.
+
+## 76.1.0
+
+* Remove `check_proxy_header_before_request` from request_helper.py since this was only used when apps
+  were deployed on the PaaS.
+
+## 76.0.2
+
+* linting changes
+
+## 76.0.1
+
+* Reject Gibraltar’s postcode (`GX11 1AA) when validating postal addresses
+
+## 76.0.0
+
+* Remove use of `NOTIFY_RUNTIME_PLATFORM` and `NOTIFY_LOG_PATH` flask config parameters, which no longer did anything. Technically this only affects users if the consumed the paramter themselves and relied on utils code setting default values for them.
+
+## 75.2.0
+
+* Add `InsensitiveSet` class (behaves like a normal set, but with uniqueness determined by normalised values)
+
+## 75.1.1
+
+* Don't set `statsd_host` in `set_gunicorn_defaults` - not all apps have statsd.
+
+## 75.1.0
+
+* Split `logging.formatting` submodule out of `logging` module. All components should remain
+  accessible via the `logging` module, so this shouldn't affect existing code.
+* Introduce `gunicorn_defaults` module.
+
 ## 75.0.0
 
 * BREAKING CHANGE: notifications_utils/clients/encryption/encryption_client has been removed. It

MANIFEST.in

@@ -2,3 +2,6 @@ recursive-include notifications_utils *.jinja2
 recursive-include notifications_utils *.json
 recursive-include notifications_utils *.txt
 include notifications_utils/international_billing_rates.yml
+include notifications_utils/version_tools/.pre-commit-config.yaml
+include notifications_utils/version_tools/pyproject.toml
+include notifications_utils/version_tools/requirements_for_test_common.in

Makefile

@@ -5,9 +5,15 @@ SHELL := /bin/bash
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
+.PHONY: freeze-requirements
+freeze-requirements: ## Pin all test requirements including sub dependencies into requirements_for_test.txt
+	pip install --upgrade pip-tools
+	pip-compile requirements_for_test.in setup.py --output-file requirements_for_test.txt
+
 .PHONY: bootstrap
 bootstrap: ## Build project
 	pip install -r requirements_for_test.txt
+	pip install -e .
 
 .PHONY: test
 test: ## Run tests
@@ -16,6 +22,10 @@ test: ## Run tests
 	pytest -n auto
 	python setup.py sdist
 
+.PHONY: watch-tests
+watch-tests: ## Automatically rerun tests
+	ptw --runner "pytest --testmon -n auto"
+
 clean:
 	rm -rf cache venv
 

notifications_utils/__init__.py

@@ -3,15 +3,6 @@
 SMS_CHAR_COUNT_LIMIT = 918  # 153 * 6, no network issues but check with providers before upping this further
 LETTER_MAX_PAGE_COUNT = 10
 
-# regexes for use in recipients.validate_email_address.
-# Valid characters taken from https://en.wikipedia.org/wiki/Email_address#Local-part
-# Note: Normal apostrophe eg `Firstname-o'[email protected]` is allowed.
-# hostname_part regex: xn in regex signifies possible punycode conversions, which would start `xn--`;
-# the hyphens are matched for later in the regex.
-hostname_part = re.compile(r"^(xn|[a-z0-9]+)(-?-[a-z0-9]+)*$", re.IGNORECASE)
-tld_part = re.compile(r"^([a-z]{2,63}|xn--([a-z0-9]+-)*[a-z0-9]+)$", re.IGNORECASE)
-VALID_LOCAL_CHARS = r"a-zA-Z0-9.!#$%&'*+/=?^_`{|}~\-"
-EMAIL_REGEX_PATTERN = r"^[{}]+@([^.@][^@\s]+)$".format(VALID_LOCAL_CHARS)
 email_with_smart_quotes_regex = re.compile(
     # matches wider than an email - everything between an at sign and the nearest whitespace
     r"(^|\s)\S+@\S+(\s|$)",