From 88363b543f6f963247c332e9d7830bc782ed6e2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Torbj=C3=B8rn=20Vatn?= <torbjorn.vatn@unacast.com>
Date: Tue, 21 Jun 2022 11:24:13 +0200
Subject: [PATCH] Switch Markdown engine to markdown-it-py (#19702)

---
 airflow/www/static/css/main.css |   4 ++
 airflow/www/utils.py            |   5 +-
 setup.cfg                       |   3 +
 tests/www/test_utils.py         | 112 ++++++++++++++++++++++++++------
 4 files changed, 103 insertions(+), 21 deletions(-)

diff --git a/airflow/www/static/css/main.css b/airflow/www/static/css/main.css
index 05eda9d42aa04..e735b7b9bb46b 100644
--- a/airflow/www/static/css/main.css
+++ b/airflow/www/static/css/main.css
@@ -465,6 +465,10 @@ label[for="timezone-other"],
   z-index: 1070;
 }
 
+details summary {
+  display: list-item;
+}
+
 .menu-scroll {
   max-height: 300px;
   overflow-y: auto;
diff --git a/airflow/www/utils.py b/airflow/www/utils.py
index 2516e9108a344..c8b97ec901538 100644
--- a/airflow/www/utils.py
+++ b/airflow/www/utils.py
@@ -21,7 +21,6 @@
 from typing import Any, Dict, List, Optional, Union
 from urllib.parse import urlencode
 
-import markdown
 import sqlalchemy as sqla
 from flask import Response, request, url_for
 from flask.helpers import flash
@@ -31,6 +30,7 @@
 from flask_appbuilder.models.sqla.filters import get_field_setup_query, set_value_to_type
 from flask_appbuilder.models.sqla.interface import SQLAInterface
 from flask_babel import lazy_gettext
+from markdown_it import MarkdownIt
 from markupsafe import Markup
 from pendulum.datetime import DateTime
 from pygments import highlight, lexers
@@ -476,10 +476,11 @@ def json_render(obj, lexer):
 
 def wrapped_markdown(s, css_class='rich_doc'):
     """Convert a Markdown string to HTML."""
+    md = MarkdownIt("gfm-like")
     if s is None:
         return None
     s = textwrap.dedent(s)
-    return Markup(f'<div class="{css_class}" >' + markdown.markdown(s, extensions=['tables']) + "</div>")
+    return Markup(f'<div class="{css_class}" >{md.render(s)}</div>')
 
 
 def get_attr_renderer():
diff --git a/setup.cfg b/setup.cfg
index e0976f9ba3b9b..1512a6201c46e 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -135,6 +135,7 @@ install_requires =
     # we pin to the same upper-bound as connexion.
     jsonschema>=3.2.0, <5.0
     lazy-object-proxy
+    linkify-it-py>=2.0.0
     lockfile>=0.12.2
     markdown>=3.0
     # Markupsafe 2.1.0 breaks with error: import name 'soft_unicode' from 'markupsafe'.
@@ -142,8 +143,10 @@ install_requires =
     # https://github.com/pallets/markupsafe/issues/284
     # or when we will be able to upgrade JINJA to newer version (currently limited due to Flask and
     # Flask Application Builder)
+    markdown-it-py>=2.1.0
     markupsafe>=1.1.1,<2.1.0
     marshmallow-oneofschema>=2.0.1
+    mdit-py-plugins>=0.3.0
     packaging>=14.0
     pathspec~=0.9.0
     pendulum>=2.0
diff --git a/tests/www/test_utils.py b/tests/www/test_utils.py
index 01c49e1fdcafd..8ac06f75de010 100644
--- a/tests/www/test_utils.py
+++ b/tests/www/test_utils.py
@@ -184,29 +184,54 @@ def test_markdown_none(self):
 class TestWrappedMarkdown(unittest.TestCase):
     def test_wrapped_markdown_with_docstring_curly_braces(self):
         rendered = wrapped_markdown("{braces}", css_class="a_class")
-        assert '<div class="a_class" ><p>{braces}</p></div>' == rendered
+        assert (
+            '''<div class="a_class" ><p>{braces}</p>
+</div>'''
+            == rendered
+        )
 
     def test_wrapped_markdown_with_some_markdown(self):
-        rendered = wrapped_markdown("*italic*\n**bold**\n", css_class="a_class")
+        rendered = wrapped_markdown(
+            """*italic*
+        **bold**
+        """,
+            css_class="a_class",
+        )
+
         assert (
             '''<div class="a_class" ><p><em>italic</em>
-<strong>bold</strong></p></div>'''
+<strong>bold</strong></p>
+</div>'''
             == rendered
         )
 
     def test_wrapped_markdown_with_table(self):
         rendered = wrapped_markdown(
-            """| Job | Duration |
-               | ----------- | ----------- |
-               | ETL | 14m |"""
+            """
+| Job | Duration |
+| ----------- | ----------- |
+| ETL | 14m |
+"""
         )
 
         assert (
-            '<div class="rich_doc" ><table>\n<thead>\n<tr>\n<th>Job</th>\n'
-            '<th>Duration</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>ETL'
-            '</td>\n<td>14m</td>\n</tr>\n</tbody>\n'
-            '</table></div>'
-        ) == rendered
+            '''<div class="rich_doc" ><table>
+<thead>
+<tr>
+<th>Job</th>
+<th>Duration</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>ETL</td>
+<td>14m</td>
+</tr>
+</tbody>
+</table>
+</div>'''
+            == rendered
+        )
 
     def test_wrapped_markdown_with_indented_lines(self):
         rendered = wrapped_markdown(
@@ -217,7 +242,11 @@ def test_wrapped_markdown_with_indented_lines(self):
             """
         )
 
-        assert '<div class="rich_doc" ><h1>header</h1>\n<p>1st line\n2nd line</p></div>' == rendered
+        assert (
+            '''<div class="rich_doc" ><h1>header</h1>\n<p>1st line\n2nd line</p>
+</div>'''
+            == rendered
+        )
 
     def test_wrapped_markdown_with_raw_code_block(self):
         rendered = wrapped_markdown(
@@ -235,10 +264,12 @@ def test_wrapped_markdown_with_raw_code_block(self):
         )
 
         assert (
-            '<div class="rich_doc" ><h1>Markdown code block</h1>\n'
-            '<p>Inline <code>code</code> works well.</p>\n'
-            '<pre><code>Code block\ndoes not\nrespect\nnewlines\n</code></pre></div>'
-        ) == rendered
+            '''<div class="rich_doc" ><h1>Markdown code block</h1>
+<p>Inline <code>code</code> works well.</p>
+<pre><code>Code block\ndoes not\nrespect\nnewlines\n</code></pre>
+</div>'''
+            == rendered
+        )
 
     def test_wrapped_markdown_with_nested_list(self):
         rendered = wrapped_markdown(
@@ -251,6 +282,49 @@ def test_wrapped_markdown_with_nested_list(self):
         )
 
         assert (
-            '<div class="rich_doc" ><h3>Docstring with a code block</h3>\n'
-            '<ul>\n<li>And<ul>\n<li>A nested list</li>\n</ul>\n</li>\n</ul></div>'
-        ) == rendered
+            '''<div class="rich_doc" ><h3>Docstring with a code block</h3>
+<ul>
+<li>And
+<ul>
+<li>A nested list</li>
+</ul>
+</li>
+</ul>
+</div>'''
+            == rendered
+        )
+
+    def test_wrapped_markdown_with_collapsible_section(self):
+        rendered = wrapped_markdown(
+            """
+# A collapsible section with markdown
+<details>
+  <summary>Click to expand!</summary>
+
+  ## Heading
+  1. A numbered
+  2. list
+     * With some
+     * Sub bullets
+</details>
+            """
+        )
+
+        assert (
+            '''<div class="rich_doc" ><h1>A collapsible section with markdown</h1>
+<details>
+  <summary>Click to expand!</summary>
+<h2>Heading</h2>
+<ol>
+<li>A numbered</li>
+<li>list
+<ul>
+<li>With some</li>
+<li>Sub bullets</li>
+</ul>
+</li>
+</ol>
+</details>
+</div>'''
+            == rendered
+        )