fix: handle unwrapped MD objects in docstrings (#2269)

CHANGELOG.md

@@ -3,6 +3,12 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Version [v1.0.1] - 2023-09-18
+
+### Fixed
+
+* Docstring with an unwrapped `Markdown.MD` object, such as the ones created when the `Markdown.@doc_str` macro is used, are correctly handled again. ([#2269])
+
 ## Version [v1.0.0] - 2023-09-15
 
 ### Version changes
@@ -1237,6 +1243,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [v0.27.24]: https://github.com/JuliaDocs/Documenter.jl/releases/tag/v0.27.24
 [v0.27.25]: https://github.com/JuliaDocs/Documenter.jl/releases/tag/v0.27.25
 [v1.0.0]: https://github.com/JuliaDocs/Documenter.jl/releases/tag/v1.0.0
+[v1.0.1]: https://github.com/JuliaDocs/Documenter.jl/releases/tag/v1.0.1
 [#198]: https://github.com/JuliaDocs/Documenter.jl/issues/198
 [#245]: https://github.com/JuliaDocs/Documenter.jl/issues/245
 [#487]: https://github.com/JuliaDocs/Documenter.jl/issues/487
@@ -1675,6 +1682,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [#2252]: https://github.com/JuliaDocs/Documenter.jl/issues/2252
 [#2259]: https://github.com/JuliaDocs/Documenter.jl/issues/2259
 [#2260]: https://github.com/JuliaDocs/Documenter.jl/issues/2260
+[#2269]: https://github.com/JuliaDocs/Documenter.jl/issues/2269
 [JuliaLang/julia#29344]: https://github.com/JuliaLang/julia/issues/29344
 [JuliaLang/julia#36953]: https://github.com/JuliaLang/julia/issues/36953
 [JuliaLang/julia#38054]: https://github.com/JuliaLang/julia/issues/38054

src/DocSystem.jl

@@ -280,21 +280,42 @@ category(::Module) = :module
 category(::Any) = :constant
 
 """
-    DocSystem.parsedoc(docstr::DocStr)
+    DocSystem.parsedoc(docstr::DocStr) -> Markdown.MD
 
 Thin internal wrapper around `Base.Docs.parsedoc` which prints additional debug information
 in case `Base.Docs.parsedoc` fails with an exception.
 """
 function parsedoc(docstr::DocStr)
-    try
-        Base.Docs.parsedoc(docstr)
+    md = try
+        Base.Docs.parsedoc(docstr) :: Markdown.MD
     catch exception
         @error """
         parsedoc failed to parse a docstring into Markdown. This indicates a problem with the docstring.
         """ exception docstr.data collect(docstr.text) docstr.object
         # Note: collect is there because svec does not print as nicely as a vector
         rethrow(exception)
     end
+    # Normally, the docsystem double wraps the docstrings in Markdown.MD, and so we need to unwrap
+    # it. _However_, if the MD object is attached directly with the @doc macro, which can happen,
+    # for example, when using the @doc_str macro, i.e.
+    #
+    #   @doc doc"""
+    #   ...
+    #   """ function foo end
+    #
+    # Then it does _not_ get double wrapped. So what we promise here is that DocSystem.parsedoc
+    # will return the unwrapped Markdown.MD object, which we can e.g. pass to MarkdownAST conversion
+    # directly. But we need to check if it actually is double wrapped or not.
+    #
+    # This heuristic should work for checking the double wrapping:
+    while length(md.content) == 1 && isa(first(md.content), Markdown.MD)
+        inner_md = only(md.content)
+        # The docstring's outer Markdown.MD contains necessary metadata, however, so we need to
+        # retain it.
+        inner_md.meta = md.meta
+        md = inner_md
+    end
+    return md
 end
 
 end

src/doctests.jl

@@ -59,13 +59,7 @@ function _doctest(page::Documenter.Page, doc::Documenter.Document)
 end
 
 function _doctest(docstr::Docs.DocStr, mod::Module, doc::Documenter.Document)
-    md = DocSystem.parsedoc(docstr)
-    # Note: parsedocs / formatdoc in Base is weird. It double-wraps the docstring Markdown
-    # in a Markdown.MD object..
-    @assert isa(md, Markdown.MD) # relying on Julia internals here
-    while length(md.content) == 1 && isa(first(md.content), Markdown.MD)
-        md = first(md.content)
-    end
+    md::Markdown.MD = DocSystem.parsedoc(docstr)
     mdast = try
         convert(MarkdownAST.Node, md)
     catch err

src/expander_pipeline.jl

@@ -565,7 +565,7 @@ function Selectors.runner(::Type{Expanders.AutoDocsBlocks}, node, page, doc)
                     """)
                 continue
             end
-            markdown = Documenter.DocSystem.parsedoc(docstr)
+            markdown::Markdown.MD = Documenter.DocSystem.parsedoc(docstr)
             docsnode = create_docsnode([markdown], [docstr], object, page, doc)
 
             # Track the order of insertion of objects per-binding.
@@ -963,8 +963,7 @@ function create_docsnode(docstrings, results, object, page, doc)
     docsnode = DocsNode(anchor, object, page)
     # Convert docstring to MarkdownAST, convert Heading elements, and push to DocsNode
     for (markdown, result) in zip(docstrings, results)
-        # parsedoc() does this double MD wrapping..
-        ast = convert(Node, markdown.content[1])
+        ast = convert(Node, markdown)
         doc.user.highlightsig && highlightsig!(ast)
         # The following 'for' corresponds to the old dropheaders() function
         for headingnode in ast.children

test/docsystem.jl

@@ -27,6 +27,10 @@ module TestDocstrings
 
     "baz(::Float64)"
     baz(::Float64)
+
+    using Markdown: @doc_str
+    @doc doc"qux(::Float64)"
+    qux(::Float64)
 end
 
 @testset "DocSystem" begin

test/examples/make.jl

@@ -138,7 +138,8 @@ module AutoDocs
         "random constant"
         qq = 3.14
 
-        "random function"
+        using Markdown: @doc_str
+        @doc doc"random function"
         function qqq end
     end
 end