feat: upgrade pinned requirements to click 8+

We were forced to pin click to < v8 because of missing dependencies. In
particular, click_repl was broken. This is no longer the case, as click_repl
0.20 was published. Also, Jinja2 now includes type annotations, which allows us
to get rid of a few "# type: ignore" statements.

We take the opportunity to upgrade all requirements, which allows us resolve a
security issue on urllib3<1.26.0.

requirements/base.in

@@ -1,13 +1,8 @@
 appdirs
-click>=7.0,<8.0
+click
 click_repl
 mypy
 pycryptodome
-jinja2>=2.9,<3.0
+jinja2
 kubernetes
 pyyaml>=4.2b1
-
-# Add constraints until we are compatible with click 8.0
-markupsafe<2.0
-# Installing urllib3==1.26.0 causes compatibility errors with requests==2.24.0
-urllib3<1.26.0

requirements/base.txt

@@ -8,35 +8,33 @@ appdirs==1.4.4
     # via -r requirements/base.in
 cachetools==4.2.2
     # via google-auth
-certifi==2020.12.5
+certifi==2021.5.30
     # via
     #   kubernetes
     #   requests
 chardet==4.0.0
     # via requests
-click-repl==0.1.6
+click-repl==0.2.0
     # via -r requirements/base.in
-click==7.1.2
+click==8.0.1
     # via
     #   -r requirements/base.in
     #   click-repl
-google-auth==1.30.0
+google-auth==1.30.1
     # via kubernetes
 idna==2.10
     # via requests
-jinja2==2.11.3
+jinja2==3.0.1
     # via -r requirements/base.in
-kubernetes==12.0.1
+kubernetes==17.17.0
     # via -r requirements/base.in
-markupsafe==1.1.1
-    # via
-    #   -r requirements/base.in
-    #   jinja2
+markupsafe==2.0.1
+    # via jinja2
 mypy-extensions==0.4.3
     # via mypy
 mypy==0.812
     # via -r requirements/base.in
-oauthlib==3.1.0
+oauthlib==3.1.1
     # via requests-oauthlib
 prompt-toolkit==3.0.18
     # via click-repl
@@ -68,19 +66,17 @@ six==1.16.0
     #   google-auth
     #   kubernetes
     #   python-dateutil
-    #   websocket-client
 typed-ast==1.4.3
     # via mypy
 typing-extensions==3.10.0.0
     # via mypy
-urllib3==1.25.11
+urllib3==1.26.5
     # via
-    #   -r requirements/base.in
     #   kubernetes
     #   requests
 wcwidth==0.2.5
     # via prompt-toolkit
-websocket-client==0.59.0
+websocket-client==1.0.1
     # via kubernetes
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/dev.txt

@@ -12,15 +12,15 @@ appdirs==1.4.4
     #   black
 astroid==2.5.6
     # via pylint
-black==21.5b1
+black==21.5b2
     # via -r requirements/dev.in
 bleach==3.3.0
     # via readme-renderer
 cachetools==4.2.2
     # via
     #   -r requirements/base.txt
     #   google-auth
-certifi==2020.12.5
+certifi==2021.5.30
     # via
     #   -r requirements/base.txt
     #   kubernetes
@@ -31,9 +31,9 @@ chardet==4.0.0
     # via
     #   -r requirements/base.txt
     #   requests
-click-repl==0.1.6
+click-repl==0.2.0
     # via -r requirements/base.txt
-click==7.1.2
+click==8.0.1
     # via
     #   -r requirements/base.txt
     #   black
@@ -45,35 +45,33 @@ cryptography==3.4.7
     # via secretstorage
 docutils==0.17.1
     # via readme-renderer
-google-auth==1.30.0
+google-auth==1.30.1
     # via
     #   -r requirements/base.txt
     #   kubernetes
 idna==2.10
     # via
     #   -r requirements/base.txt
     #   requests
-importlib-metadata==4.0.1
+importlib-metadata==4.4.0
     # via
     #   keyring
-    #   pep517
-    #   pyinstaller
     #   twine
 isort==5.8.0
     # via pylint
 jeepney==0.6.0
     # via
     #   keyring
     #   secretstorage
-jinja2==2.11.3
+jinja2==3.0.1
     # via -r requirements/base.txt
 keyring==23.0.1
     # via twine
-kubernetes==12.0.1
+kubernetes==17.17.0
     # via -r requirements/base.txt
 lazy-object-proxy==1.6.0
     # via astroid
-markupsafe==1.1.1
+markupsafe==2.0.1
     # via
     #   -r requirements/base.txt
     #   jinja2
@@ -86,7 +84,7 @@ mypy-extensions==0.4.3
     #   mypy
 mypy==0.812
     # via -r requirements/base.txt
-oauthlib==3.1.0
+oauthlib==3.1.1
     # via
     #   -r requirements/base.txt
     #   requests-oauthlib
@@ -123,7 +121,7 @@ pyinstaller-hooks-contrib==2021.1
     # via pyinstaller
 pyinstaller==4.3
     # via -r requirements/dev.in
-pylint==2.8.2
+pylint==2.8.3
     # via -r requirements/dev.in
 pyparsing==2.4.7
     # via packaging
@@ -169,29 +167,24 @@ six==1.16.0
     #   kubernetes
     #   python-dateutil
     #   readme-renderer
-    #   websocket-client
 toml==0.10.2
     # via
     #   black
     #   pep517
     #   pylint
-tqdm==4.60.0
+tqdm==4.61.0
     # via twine
 twine==3.4.1
     # via -r requirements/dev.in
 typed-ast==1.4.3
     # via
     #   -r requirements/base.txt
-    #   astroid
-    #   black
     #   mypy
 typing-extensions==3.10.0.0
     # via
     #   -r requirements/base.txt
-    #   black
-    #   importlib-metadata
     #   mypy
-urllib3==1.25.11
+urllib3==1.26.5
     # via
     #   -r requirements/base.txt
     #   kubernetes
@@ -202,16 +195,14 @@ wcwidth==0.2.5
     #   prompt-toolkit
 webencodings==0.5.1
     # via bleach
-websocket-client==0.59.0
+websocket-client==1.0.1
     # via
     #   -r requirements/base.txt
     #   kubernetes
 wrapt==1.12.1
     # via astroid
 zipp==3.4.1
-    # via
-    #   importlib-metadata
-    #   pep517
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

requirements/docs.txt

@@ -14,7 +14,7 @@ cachetools==4.2.2
     # via
     #   -r requirements/base.txt
     #   google-auth
-certifi==2020.12.5
+certifi==2021.5.30
     # via
     #   -r requirements/base.txt
     #   kubernetes
@@ -23,17 +23,17 @@ chardet==4.0.0
     # via
     #   -r requirements/base.txt
     #   requests
-click-repl==0.1.6
+click-repl==0.2.0
     # via -r requirements/base.txt
-click==7.1.2
+click==8.0.1
     # via
     #   -r requirements/base.txt
     #   click-repl
 docutils==0.16
     # via
     #   sphinx
     #   sphinx-rtd-theme
-google-auth==1.30.0
+google-auth==1.30.1
     # via
     #   -r requirements/base.txt
     #   kubernetes
@@ -43,24 +43,23 @@ idna==2.10
     #   requests
 imagesize==1.2.0
     # via sphinx
-jinja2==2.11.3
+jinja2==3.0.1
     # via
     #   -r requirements/base.txt
     #   sphinx
-kubernetes==12.0.1
+kubernetes==17.17.0
     # via -r requirements/base.txt
-markupsafe==1.1.1
+markupsafe==2.0.1
     # via
     #   -r requirements/base.txt
     #   jinja2
-    #   sphinx
 mypy-extensions==0.4.3
     # via
     #   -r requirements/base.txt
     #   mypy
 mypy==0.812
     # via -r requirements/base.txt
-oauthlib==3.1.0
+oauthlib==3.1.1
     # via
     #   -r requirements/base.txt
     #   requests-oauthlib
@@ -116,26 +115,25 @@ six==1.16.0
     #   google-auth
     #   kubernetes
     #   python-dateutil
-    #   websocket-client
 snowballstemmer==2.1.0
     # via sphinx
 sphinx-rtd-theme==0.5.2
     # via -r requirements/docs.in
-sphinx==4.0.1
+sphinx==4.0.2
     # via
     #   -r requirements/docs.in
     #   sphinx-rtd-theme
 sphinxcontrib-applehelp==1.0.2
     # via sphinx
 sphinxcontrib-devhelp==1.0.2
     # via sphinx
-sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-htmlhelp==2.0.0
     # via sphinx
 sphinxcontrib-jsmath==1.0.1
     # via sphinx
 sphinxcontrib-qthelp==1.0.3
     # via sphinx
-sphinxcontrib-serializinghtml==1.1.4
+sphinxcontrib-serializinghtml==1.1.5
     # via sphinx
 typed-ast==1.4.3
     # via
@@ -145,7 +143,7 @@ typing-extensions==3.10.0.0
     # via
     #   -r requirements/base.txt
     #   mypy
-urllib3==1.25.11
+urllib3==1.26.5
     # via
     #   -r requirements/base.txt
     #   kubernetes
@@ -154,7 +152,7 @@ wcwidth==0.2.5
     # via
     #   -r requirements/base.txt
     #   prompt-toolkit
-websocket-client==0.59.0
+websocket-client==1.0.1
     # via
     #   -r requirements/base.txt
     #   kubernetes

tests/test_env.py

@@ -20,7 +20,7 @@ def test_walk_templates_partials_are_ignored(self) -> None:
         template_name = "apps/openedx/settings/partials/common_all.py"
         renderer = env.Renderer({}, [env.TEMPLATES_ROOT], ignore_folders=["partials"])
         templates = list(renderer.walk_templates("apps"))
-        self.assertIn(template_name, renderer.environment.loader.list_templates())  # type: ignore
+        self.assertIn(template_name, renderer.environment.loader.list_templates())
         self.assertNotIn(template_name, templates)
 
     def test_is_binary_file(self) -> None:
@@ -174,5 +174,5 @@ def test_renderer_is_reset_on_config_change(self) -> None:
                 config["PLUGINS"] = ["myplugin"]
                 env2 = env.Renderer.instance(config).environment
 
-            self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates())  # type: ignore
-            self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates())  # type: ignore
+            self.assertNotIn("plugin1/myplugin.txt", env1.loader.list_templates())
+            self.assertIn("plugin1/myplugin.txt", env2.loader.list_templates())

tutor/env.py

@@ -15,6 +15,14 @@
 BIN_FILE_EXTENSIONS = [".ico", ".jpg", ".png", ".ttf", ".woff", ".woff2"]
 
 
+class JinjaEnvironment(jinja2.Environment):
+    loader: jinja2.BaseLoader
+
+    def __init__(self, template_roots: List[str]) -> None:
+        loader = jinja2.FileSystemLoader(template_roots)
+        super().__init__(loader=loader, undefined=jinja2.StrictUndefined)
+
+
 class Renderer:
     @classmethod
     def instance(cls: Type["Renderer"], config: Config) -> "Renderer":
@@ -39,10 +47,7 @@ def __init__(
         self.ignore_folders.append(".git")
 
         # Create environment
-        environment = jinja2.Environment(
-            loader=jinja2.FileSystemLoader(template_roots),
-            undefined=jinja2.StrictUndefined,
-        )
+        environment = JinjaEnvironment(template_roots)
         environment.filters["common_domain"] = utils.common_domain
         environment.filters["encrypt"] = utils.encrypt
         environment.filters["list_if"] = utils.list_if
@@ -61,9 +66,7 @@ def iter_templates_in(self, *prefix: str) -> Iterable[str]:
         The elements of `prefix` must contain only "/", and not os.sep.
         """
         full_prefix = "/".join(prefix)
-        env_templates: List[
-            str
-        ] = self.environment.loader.list_templates()  # type:ignore[no-untyped-call]
+        env_templates: List[str] = self.environment.loader.list_templates()
         for template in env_templates:
             if template.startswith(full_prefix) and self.is_part_of_env(template):
                 yield template