Merge pull request #4220 from magfest/add-forms-documentation

Overhaul new form validations

uber/api.py

@@ -744,7 +744,7 @@ def export_attendees(self, id, full=False, include_group=False):
             if not account:
                 raise HTTPError(404, 'No attendee account found with this ID')
 
-            attendees_to_export = account.valid_attendees if include_group else [a for a in account.attendees if not a.group]
+            attendees_to_export = account.valid_attendees if include_group else [a for a in account.valid_attendees if not a.group]
 
             attendees = _prepare_attendees_export(attendees_to_export, include_apps=full)
             return {

uber/config.py

@@ -5,6 +5,7 @@
 import os
 import pytz
 import re
+import redis
 import uuid
 from collections import defaultdict, OrderedDict
 from datetime import date, datetime, time, timedelta
@@ -1000,6 +1001,9 @@ def _unrepr(d):
 _unrepr(_config['appconf'])
 c.APPCONF = _config['appconf'].dict()
 c.SENTRY = _config['sentry'].dict()
+c.REDISCONF = _config['redis'].dict()
+
+c.REDIS_STORE = redis.Redis(host=c.REDISCONF['host'], port=c.REDISCONF['port'], db=c.REDISCONF['db'], decode_responses=True)
 
 c.BADGE_PRICES = _config['badge_prices']
 for _opt, _val in chain(_config.items(), c.BADGE_PRICES.items()):

uber/configspec.ini

@@ -1850,6 +1850,11 @@ environment = string(default="production")
 sample_rate = integer(default=100)
 release = string(default="")
 
+[redis]
+host = string(default="localhost")
+port = integer(default=6379)
+db = integer(default=0)
+
 [appconf]
 # This is all CherryPy configuration.
 

uber/decorators.py

@@ -726,27 +726,6 @@ def wrapper(func):
 validation, prereg_validation = Validation(), Validation()
 
 
-class WTFormValidation:
-    def __init__(self):
-        self.validations = defaultdict(OrderedDict)
-
-    def __getattr__(self, field_name):
-        def wrapper(func):
-            self.validations[field_name][func.__name__] = func
-            return func
-        return wrapper
-
-    def get_validations_by_field(self, field_name):
-        field_validations = self.validations.get(field_name)
-        return list(field_validations.values()) if field_validations else []
-
-    def get_validation_dict(self):
-        all_validations = {}
-        for key, dict in self.validations.items():
-            all_validations[key] = list(dict.values())
-        return all_validations
-
-
 class ReceiptItemConfig:
     def __init__(self):
         self.items = defaultdict(OrderedDict)

uber/forms/README.md

@@ -0,0 +1,122 @@
+# Form Guide for Ubersystem/RAMS
+NOTE: this guide is currently being built in Notion, below is a rough draft.
+
+Forms represent the vast majority of attendees' (and many admins'!) interaction with Ubersystem. As such, they are highly dependent on business logic and are often complex. A single field may be required under some conditions but not others, change its labeling in some contexts, show help text to attendees but not admins, etc. This guide is to help you understand, edit, and override forms without creating a giant mess. Hopefully.
+
+## Forms Are a WIP
+Up until the writing of this guide, all forms in Ubersystem were driven entirely by Jinja2 macros, jQuery, and HTML, with form handling done largely inside individual page handlers (with the exception of validations, which were all in **model_checks.py**). As of 2023, *attendee* and *group* forms are the only forms that use the technologies and conventions described below, unless otherwise noted. Conversion of other forms is ongoing, and help with those conversions is extremely welcome.
+
+## How Forms Are Built
+We rely on the following frameworks and modules for our forms:
+- [WTForms](https://wtforms.readthedocs.io/en/3.0.x/) defines our forms as declarative data, along with many of their static properties. Each set of forms is organized in one file per type of entity, similar to our **models** folder, and they are found in **uber/forms/**. Inherited classes and other WTForms customizations are in **uber/forms/__init__.py**.
+- [Jinja2](https://jinja.palletsprojects.com/en/3.1.x/templates/) provides *macros* that render the scaffolding for fields (these macros call WTForms to render the fields themselves) and *blocks* that define sections of forms for appending fields or overriding.
+  - Form macros are found in **uber/template/forms/macros.html** -- always use these macros rather than writing your own scaffolding.
+- [Bootstrap 5](https://getbootstrap.com/docs/5.0/) provides the styling and responsive layout for forms. Always use the grid layout ("col" divs contained inside "row" divs) when adding fields.
+
+## Form Validations
+There are three broad categories of validations:
+- Field-level validations: simple data validations, often [WTForm built-in validations](https://wtforms.readthedocs.io/en/3.0.x/validators/), which are passed to field constructors. These validations only have access to their form's data. Additional field validations are implemented using the `CustomValidation()`
+- Custom validations: validations that require knowledge of the model or use complex calculations, and in some cases don't even correspond to any fields. For example, we check and make sure attendees don't have so many badge extras that their total is above $9,999, as that is Stripe's limit.
+- "New or changed" validations: these are custom validations that only run if a specific field's data changed or if a model has not been saved to the database yet. For example, we check that a selected preordered merch package is still available, but only if the preordered merch field changed or the attendee is new.
+
+### Some caveats
+WTForms has a way to add custom validations to any field by adding a `validate_fieldname` function to a form. We avoid using this because it can only return one error, which is poor UX, and because it is difficult to override these validations in event plugins.
+
+Field-level validations have an added bonus of rendering matching clientside validations where possible. However, we actually skip running clientside validations on our forms, again because they only show only one error at a time. Since we use AJAX for server validations and display all errors at once, clientside validations are rendered moot.
+
+### Field-Level Validations
+Adding a field-level validation involves simply passing the validator class(es) in a list through the field constructor's `validators` parameter. Let's take this `email` field as an example:
+```
+email = EmailField('Email Address', validators=[
+    validators.InputRequired("Please enter an email address."),
+    validators.Length(max=255, message="Email addresses cannot be longer than 255 characters."),
+    validators.Email(granular_message=True),
+    ],
+    render_kw={'placeholder': '[email protected]'})
+```
+This field has three field validations: a validator that requires the field not be empty, a validator that limits the email address to 255 characters, and a validator that checks the email address using the [email_validator package](https://pypi.org/project/email-validator/) and passes through the exact error message if the email fails validation.
+
+For more about what validators are available, see [WTForms' documentation on built-in validators](https://wtforms.readthedocs.io/en/3.0.x/validators/). We may also at some point add our own field-level validators -- if so, they should be in **forms/validations.py**.
+
+### Selectively Required Fields and `get_optional_fields`
+Almost all required fields should have the `InputRequired` validator passed to their constructor rather than a custom validator to check their data. However, many fields are optional in certain circumstances, usually based on the model's current state. The special function `get_optional_fields` bridges this gap.
+
+Let's look at a simple example of this function for a group.
+```
+class TableInfo(GroupInfo):
+    name = StringField('Table Name', validators=[
+        validators.InputRequired("Please enter a table name."),
+        validators.Length(max=40, message="Table names cannot be longer than 40 characters.")
+        ])
+    description = StringField('Description', validators=[
+        validators.InputRequired("Please provide a brief description of your business.")
+        ], description="Please keep to one sentence.")
+    // other fields
+
+    def get_optional_fields(self, group, is_admin=False):
+        optional_list = super().get_optional_fields(group, is_admin)
+        if not group.is_dealer:
+            optional_list.extend(
+                ['description', 'website', 'wares', 'categories', 'address1', 'city', 'region', 'zip_code', 'country'])
+        return optional_list
+```
+This function gets any optional fields from its parent class. Then, if the group's `is_dealer` property is false, it adds all of the form's dealer-related fields to the list of optional fields. Finally, it returns the list of optional fields. Based on this, a non-dealer group would be required to have a `name` but not a `description`.
+
+The parameters are:
+- `group`: a model object, e.g., `Attendee` or `Group`. This model will always be a "preview model" that has already had any form updates applied to it. For this reason, do _not_ check `group.is_new`, as the preview model is always "new".
+- `is_admin`: a boolean that is True if the model is being viewed in the admin area; you'll almost never need it, but there are a few cases where fields are optional for admins when they would not be optional for attendees.
+
+**NOTE**: If a field is returned by `get_optional_fields`, _all_ validations are skipped _only if_ the field is empty.
+
+### Custom and New-Or-Changed Validations
+
+
+## Adding Fields
+First, figure out if you want to add fields to an existing form or if you want to add a new form. Multiple forms can be combined and processed seamlessly on a single page, so it is good to group like fields together into their own 'forms.' Pay particular attention to which fields represent personal identifying information (PII) and group them separately from fields that don't.
+
+To declare a new form, [TODO]. To add fields to an existing form, [TODO].
+
+https://wtforms.readthedocs.io/en/3.0.x/fields/
+
+### Field Labels and Descriptions
+By default, labels and descriptions for fields are simple strings with automatic escaping for HTML/XML. Since this is not always desirable, here are a few ways to write more complex labels:
+
+- To include basic HTML (e.g., bolding or italicizing text), wrap the string in a Markup() object from the **markupsafe** library, e.g., `field_name = StringField(Markup('<b>Bold text</b>'))`
+- For complex display logic (e.g., building a label using multiple 'if' statements) add a function onto your form class named `field_name_label` or `field_name_desc`, e.g.:
+  ```
+  def pii_consent_label(self):
+    label = ''
+    # add complex display logic that modifies 'label'
+    return label
+  ```
+
+
+### Field Types
+Below is a map of what column types exist in Ubersystem models and what fields you might want to (or ought to) use when declaring the corresponding form fields.
+| Column Type | Suggested Field Type(s) |
+| --- | --- |
+| UnicodeText | StringField, TextAreaField, EmailField, TelField, PasswordField, URLField | 
+| Integer | IntegerField |
+| Date | DateField |
+| Choice | SelectField, RadioField |
+| MultiChoice | MultiSelectField |
+| Boolean | BooleanField |
+| UTCDateTime | DateTimeField, DateTimeLocalField |
+| UUID | [TODO] |
+| MutableDict | [TODO] |
+
+
+## Editing and Overriding Fields
+
+### Blocks
+
+### Change Field Name
+
+### Change Field Help Text
+
+### Troubleshooting/Dev Notes
+Deleting or adding template files requires a restart of the server.
+
+It is not currently possible to layer two plugins' block override. In other words, if you have a {% block consents %} in other_info.html in one plugin, and another {% block consents %} in other_info.html in another plugin, the last plugin loaded will override the first plugin's consents block. This is considered an edge case and fixing it is not currently a priority.
+
+There are some weird behaviors if you apply the Markup() class to a description with a popup link inside it. If you're encountering this, try to apply Markup() to the rest of the text, then append the popup link -- that should work.

uber/forms/__init__.py

@@ -1,6 +1,7 @@
 import re
 import cherrypy
 
+from collections import defaultdict, OrderedDict
 from importlib import import_module
 from markupsafe import Markup
 from wtforms import Form, StringField, SelectField, IntegerField, BooleanField, validators
@@ -81,8 +82,33 @@ def load_forms(params, model, module, form_list, prefix_dict={}, get_optional=Tr
     return form_dict
 
 
+class CustomValidation:
+    def __init__(self):
+        self.validations = defaultdict(OrderedDict)
+
+    def __bool__(self):
+        return bool(self.validations)
+
+    def __getattr__(self, field_name):
+        def wrapper(func):
+            self.validations[field_name][func.__name__] = func
+            return func
+        return wrapper
+
+    def get_validations_by_field(self, field_name):
+        field_validations = self.validations.get(field_name)
+        return list(field_validations.values()) if field_validations else []
+
+    def get_validation_dict(self):
+        all_validations = {}
+        for key, dict in self.validations.items():
+            all_validations[key] = list(dict.values())
+        return all_validations
+
+
 class MagForm(Form):
     field_aliases = {}
+    field_validation, new_or_changed_validation = CustomValidation(), CustomValidation()
 
     def get_optional_fields(self, model, is_admin=False):
         return []
@@ -342,4 +368,4 @@ def getlist(self, arg):
 
 
 from uber.forms.attendee import *  # noqa: F401,E402,F403
-from uber.forms.group import *  # noqa: F401,E402,F403
+from uber.forms.group import *  # noqa: F401,E402,F403