HTML: split between custom and common filters

...and fix imports (always absolute)

src/formats/cherrytree.py

@@ -9,7 +9,7 @@
 import common
 import converter
 import intermediate_format as imf
-import markdown_lib
+import markdown_lib.common
 
 
 LOGGER = logging.getLogger("jimmy")

src/formats/dynalist.py

@@ -5,7 +5,7 @@
 import common
 import converter
 import intermediate_format as imf
-import markdown_lib
+import markdown_lib.common
 
 
 def handle_markdown_links(body: str, root_folder: Path) -> imf.NoteLinks:

src/formats/joplin.py

@@ -9,7 +9,7 @@
 import common
 import converter
 import intermediate_format as imf
-import markdown_lib
+import markdown_lib.common
 
 
 class ItemType(enum.IntEnum):

src/formats/nimbus_note.py

@@ -3,13 +3,11 @@
 import base64
 from pathlib import Path
 
-from bs4 import BeautifulSoup
-
 import common
 import converter
 import intermediate_format as imf
 import markdown_lib.common
-import markdown_lib.html_preprocessing
+import markdown_lib.html_filter
 
 
 class Converter(converter.BaseConverter):
@@ -57,11 +55,10 @@ def convert_note(self, file_: Path, temp_folder: Path):
 
         # HTML note seems to have the name "note.html" always
         note_body_html = (temp_folder_note / "note.html").read_text(encoding="utf-8")
-
-        soup = BeautifulSoup(note_body_html, "html.parser")
-        markdown_lib.html_preprocessing.nimbus_note_streamline_lists(soup)
-
-        note_body_markdown = markdown_lib.common.markup_to_markdown(str(soup))
+        note_body_markdown = markdown_lib.common.markup_to_markdown(
+            note_body_html,
+            custom_filter=[markdown_lib.html_filter.nimbus_note_streamline_lists],
+        )
         resources = self.handle_markdown_links(note_body_markdown, temp_folder_note)
         note_imf = imf.Note(
             title,

src/formats/notion.py

@@ -6,8 +6,6 @@
 from urllib.parse import unquote
 import zipfile
 
-from bs4 import BeautifulSoup
-
 import common
 import converter
 import intermediate_format as imf
@@ -110,9 +108,9 @@ def convert_note(
             # first line is title, second is whitespace
             body = "\n".join(item.read_text(encoding="utf-8").split("\n")[2:])
         else:  # html
-            soup = BeautifulSoup(body, "html.parser")
-            markdown_lib.html_preprocessing.notion_streamline_lists(soup)
-            body = markdown_lib.common.markup_to_markdown(str(soup))
+            body = markdown_lib.common.markup_to_markdown(
+                body, custom_filter=[markdown_lib.html_filter.notion_streamline_lists]
+            )
 
         # find links
         resources, note_links = self.handle_markdown_links(body, item)

src/formats/obsidian.py

@@ -8,7 +8,7 @@
 import common
 import converter
 import intermediate_format as imf
-import markdown_lib
+import markdown_lib.common
 
 
 class Converter(converter.BaseConverter):

src/formats/rednotebook.py

@@ -8,7 +8,7 @@
 import common
 import converter
 import intermediate_format as imf
-import markdown_lib
+import markdown_lib.common
 
 
 class Converter(converter.BaseConverter):

src/formats/simplenote.py

@@ -7,7 +7,7 @@
 import common
 import converter
 import intermediate_format as imf
-import markdown_lib
+import markdown_lib.common
 
 
 class Converter(converter.BaseConverter):

src/formats/standard_notes.py

@@ -9,7 +9,7 @@
 
 import converter
 import intermediate_format as imf
-import markdown_lib
+import markdown_lib.common
 import common
 
 LOGGER = logging.getLogger("jimmy")

src/formats/synology_note_station.py

@@ -10,7 +10,8 @@
 import common
 import converter
 import intermediate_format as imf
-import markdown_lib
+import markdown_lib.common
+import markdown_lib.html_filter
 
 
 @dataclass
@@ -145,7 +146,12 @@ def convert_note(self, note_id, note_id_title_map):
 
         note_links: imf.NoteLinks = []
         if (content_html := note.get("content")) is not None:
-            content_markdown = markdown_lib.common.markup_to_markdown(content_html)
+            content_markdown = markdown_lib.common.markup_to_markdown(
+                content_html,
+                custom_filter=[
+                    markdown_lib.html_filter.synology_note_station_fix_img_src
+                ],
+            )
             # note title only needed for debug message
             body, resources_referenced, note_links = self.handle_markdown_links(
                 note["title"],

src/formats/textbundle.py

@@ -7,7 +7,7 @@
 import common
 import converter
 import intermediate_format as imf
-import markdown_lib
+import markdown_lib.common
 
 
 class Converter(converter.BaseConverter):

src/formats/zoho_notebook.py

@@ -9,7 +9,7 @@
 import common
 import converter
 import intermediate_format as imf
-import markdown_lib
+import markdown_lib.common
 
 
 class Converter(converter.BaseConverter):

src/markdown_lib/common.py

@@ -11,7 +11,7 @@
 from markdown.extensions import Extension
 import pypandoc
 
-import markdown_lib.html_preprocessing
+import markdown_lib.html_filter
 
 
 LOGGER = logging.getLogger("jimmy")
@@ -263,15 +263,17 @@ def get_inline_tags(text: str, start_characters: list[str]) -> list[str]:
 # fmt:on
 
 
-def html_to_markdown(text_html: bytes | str):
+def html_to_markdown(text_html: bytes | str, custom_filter: list | None = None):
     # some needed preprocessing
     soup = BeautifulSoup(text_html, "html.parser")
-    markdown_lib.html_preprocessing.div_checklists(soup)
-    markdown_lib.html_preprocessing.highlighting(soup)
-    markdown_lib.html_preprocessing.iframes_to_links(soup)
-    markdown_lib.html_preprocessing.streamline_tables(soup)
-    markdown_lib.html_preprocessing.synology_note_station_fix_img_src(soup)
-    markdown_lib.html_preprocessing.whitespace_in_math(soup)
+    markdown_lib.html_filter.div_checklists(soup)
+    markdown_lib.html_filter.highlighting(soup)
+    markdown_lib.html_filter.iframes_to_links(soup)
+    markdown_lib.html_filter.streamline_tables(soup)
+    markdown_lib.html_filter.whitespace_in_math(soup)
+    if custom_filter is not None:
+        for filter_ in custom_filter:
+            filter_(soup)
     text_html_filtered = str(soup)
 
     # writer: json ast -> markdown
@@ -292,7 +294,10 @@ def html_to_markdown(text_html: bytes | str):
 
 
 def markup_to_markdown(
-    text: bytes | str, format_: str = "html", resource_folder: Path = Path("tmp_media")
+    text: bytes | str,
+    format_: str = "html",
+    resource_folder: Path = Path("tmp_media"),
+    custom_filter: list | None = None,
 ) -> str:
     # Route everything through this function to get a single path of truth.
     if format_.startswith("html"):
@@ -314,7 +319,7 @@ def markup_to_markdown(
 
     # HTML filter: HTML -> filter -> HTML
     # writer: HTML -> Markdown
-    return html_to_markdown(text_html)
+    return html_to_markdown(text_html, custom_filter)
 
 
 # Problem: "//" is part of many URI (between scheme and host).

src/markdown_lib/evernote.py

@@ -10,7 +10,7 @@
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.primitives import padding
 
-import markdown_lib
+import markdown_lib.common
 
 
 # TODO: simplify