Add new stripnl option to Highlight (#2153)

Resolves #2152

docs/src/markdown/about/changelog.md

@@ -1,7 +1,9 @@
 # Changelog
 
-## 10.1.1
+## 10.2
 
+-   **NEW**: Highlight: Add new `stripnl` option to configure Pygments' default handling of stripping leading and
+    and trailing new lines from code blocks. Mainly affects fenced code blocks.
 -   **FIX**: SuperFences: Fix issue where when SuperFences attempts to test if a placeholder is its own, it can throw
     an exception.
 

docs/src/markdown/extensions/highlight.md

@@ -86,6 +86,25 @@ extend_pygments_lang = [
 `pygments_lang_class` added in 9.2.
 ///
 
+## Preserve Leading/Trailing Newlines
+
+/// new | New 10.1
+///
+
+It is not uncommon for Markdown parsers to preserve both leading and trailing newlines in things like fenced code. This
+doesn't usually happen for indented code blocks.
+
+To be clear the [SuperFences](./superfences.md) extension does not strip leading and trailing new lines, this is the
+default behavior of the Pygments syntax highlighter, not the SuperFences extension's parsing logic. When Pygments is
+disabled, both leading and trailing newlines are preserved.
+
+While it is rare for people to actually need leading and trailing newlines, if such behavior was desired, it can be
+retained by disabling the [`stripnl`](#options) option in Highlight which will ensure that all Pygments lexers have this
+option disabled.
+
+Inline code blocks, such as those provided by [InlineHilite](./inlinehilite.md), will be unaffected. Indented code
+blocks will also not be affected as leading and trailing newlines are never retained in indented code blocks.
+
 ## Line Number Styles
 
 Pygments has two available styles when outputting source code with line numbers enabled: `table` and `inline`. `table`
@@ -131,6 +150,7 @@ Option                    | Type   | Default               | Description
 `anchor_linenums`         | bool   | `#!py3 False`         | Enables the Pygments option of a similar name. If set to `#!py True`, will wrap line numbers in `#!html <a>` tags. Used in combination with `linenums` and `line_anchors`. If `line_anchors` is not configured, `__codelineno` will be assumed as the ID prefix.
 `line_anchors`            | bool   | `#!py3 False`         | Controls the Pygments option of a similar name. If set to a nonempty string, e.g. `foo`, the formatter will insert an anchor tag with an id (and name) of `foo-<code_block_number>-<line_number>`.
 `pygments_lang_class`     | bool   | `#!py3 False`         | If set to True, the language name used will be included as a class attached to the element with the associated `language_prefix`.
+`stripnl`                 | bool   | `#!py3 True`          | Strips leading and trailing newlines from code blocks. This is Pygments default behavior. Setting this to `#!py False` disables this and will retain leading and trailing newlines. This has no affect on inline code.
 
 /// new | New 7.1
 `linenums_class` was added in `7.1`.

pymdownx/__meta__.py

@@ -185,5 +185,5 @@ def parse_version(ver, pre=False):
     return Version(major, minor, micro, release, pre, post, dev)
 
 
-__version_info__ = Version(10, 1, 1, "final")
+__version_info__ = Version(10, 2, 0, "final")
 __version__ = __version_info__._get_canonical()

pymdownx/highlight.py

@@ -124,6 +124,12 @@
         False,
         'If set to True, the language name used will be included as a class attached to the element. - Defaults: False'
     ],
+    'stripnl': [
+        True,
+        'Strips leading and trailing newlines from code blocks. This is Pygments default behavior. Setting this to '
+        'False disables this and will retain leading and trailing newlines. This has no affect on inline code. '
+        '- Defaults: True'
+    ],
     '_enabled': [
         True,
         'Used internally to communicate if extension has been explicitly enabled - Default: False'
@@ -230,7 +236,7 @@ def __init__(
         noclasses=False, extend_pygments_lang=None, linenums=None, linenums_special=-1,
         linenums_style='table', linenums_class='linenums', language_prefix='language-',
         code_attr_on_pre=False, auto_title=False, auto_title_map=None, line_spans='',
-        anchor_linenums=False, line_anchors='', pygments_lang_class=False
+        anchor_linenums=False, line_anchors='', pygments_lang_class=False, stripnl=True
     ):
         """Initialize."""
 
@@ -249,6 +255,7 @@ def __init__(
         self.line_anchors = line_anchors
         self.anchor_linenums = anchor_linenums
         self.pygments_lang_class = pygments_lang_class
+        self.stripnl = stripnl
 
         if self.anchor_linenums and not self.line_anchors:
             self.line_anchors = '__codelineno'
@@ -274,15 +281,15 @@ def get_extended_language(self, language):
 
         return self.extend_pygments_lang.get(language.lower(), (language, {}))
 
-    def get_lexer(self, src, language, inline):
+    def get_lexer(self, src, language, inline, stripnl):
         """Get the Pygments lexer."""
 
         name = language
 
+        lexer_options = {'stripnl': stripnl}
         if language:
-            language, lexer_options = self.get_extended_language(language)
-        else:
-            lexer_options = {}
+            language, options = self.get_extended_language(language)
+            lexer_options.update(options)
 
         # Try and get lexer by the name given.
         try:
@@ -293,12 +300,12 @@ def get_lexer(self, src, language, inline):
         if lexer is None:
             if (self.guess_lang is True) or (self.guess_lang == 'inline' if inline else self.guess_lang == 'block'):
                 try:
-                    lexer = guess_lexer(src)
+                    lexer = guess_lexer(src, **lexer_options)
                     name = lexer.aliases[0]
                 except Exception:  # pragma: no cover
                     pass
         if lexer is None:
-            lexer = get_lexer_by_name('text')
+            lexer = get_lexer_by_name('text', **lexer_options)
             name = lexer.aliases[0]
         return lexer, name
 
@@ -324,15 +331,21 @@ def highlight(
             (self.linenums and linestart != 0) or
             (self.linenums is not False and linestart > 0)
         ) and not inline > 0
+        class_str = ''
 
         # Convert with Pygments.
         if pygments and self.use_pygments:
 
             if p_ver < (2, 12):  # pragma: no cover
                 raise RuntimeError('Pymdownx Highlight requires at least Pygments 2.12+ if enabling Pygments')
 
+            if inline:
+                stripnl = True
+            else:
+                stripnl = self.stripnl
+
             # Setup language lexer.
-            lexer, lang_name = self.get_lexer(src, language, inline)
+            lexer, lang_name = self.get_lexer(src, language, inline, stripnl)
             if self.pygments_lang_class:
                 class_names.insert(0, self.language_prefix + lang_name)
             linenums = self.linenums_style if linenums_enabled else False
@@ -499,7 +512,8 @@ def run(self, root):
                     code_attr_on_pre=self.config['code_attr_on_pre'],
                     auto_title=self.config['auto_title'],
                     auto_title_map=self.config['auto_title_map'],
-                    pygments_lang_class=self.config['pygments_lang_class']
+                    pygments_lang_class=self.config['pygments_lang_class'],
+                    stripnl=self.config['stripnl']
                 )
                 placeholder = self.md.htmlStash.store(
                     code.highlight(

pymdownx/superfences.py

@@ -392,6 +392,7 @@ def get_hl_settings(self):
             self.line_anchors = config.get('line_anchors', '')
             self.anchor_linenums = config.get('anchor_linenums', False)
             self.pygments_lang_class = config.get('pygments_lang_class', False)
+            self.stripnl = config.get('stripnl', True)
 
     def clear(self):
         """Reset the class variables."""
@@ -795,7 +796,8 @@ def highlight(self, src="", language="", options=None, md=None, **kwargs):
             line_spans=self.line_spans,
             line_anchors=self.line_anchors,
             anchor_linenums=self.anchor_linenums,
-            pygments_lang_class=self.pygments_lang_class
+            pygments_lang_class=self.pygments_lang_class,
+            stripnl=self.stripnl
         ).highlight(
             src,
             language,

tests/test_extensions/test_superfences.py

@@ -111,6 +111,42 @@ def test_title_table(self):
         )
 
 
+class TestHighlightStrip(util.MdCase):
+    """Test Highlight's stripping of new lines."""
+
+    extension = ['pymdownx.highlight', 'pymdownx.superfences']
+    extension_configs = {'pymdownx.highlight': {'stripnl': False}}
+
+    def test_no_stripnl(self):
+        """Test no stripping of leading and trailing new lines."""
+
+        self.check_markdown(
+            r'''
+            ```py
+
+            import foo
+
+
+            import bar
+
+
+
+            ```
+            ''',
+            r'''
+            <div class="highlight"><pre><span></span><code>
+            <span class="kn">import</span> <span class="nn">foo</span>
+
+
+            <span class="kn">import</span> <span class="nn">bar</span>
+
+
+            </code></pre></div>
+            ''',
+            True
+        )
+
+
 class TestHighlightAutoTitleOverride(util.MdCase):
     """Test title cases."""