From 4916589fd96717046af03dc3a80da4db9f7cb311 Mon Sep 17 00:00:00 2001
From: Tim Holy <tim.holy@gmail.com>
Date: Tue, 31 Dec 2019 07:00:34 -0600
Subject: [PATCH 1/3] Support brief and extended docs (closes #25930)

A level-1 header "Extended help" indicates that the content that follows
requires `??foo` rather than just `?foo`.
---
 NEWS.md                                       |  8 +++++
 doc/src/manual/documentation.md               |  4 +++
 stdlib/Markdown/src/Markdown.jl               | 26 +++++++++++++++++
 stdlib/Markdown/src/parse/parse.jl            |  7 +++++
 stdlib/Markdown/src/render/terminal/render.jl |  4 +++
 stdlib/Markdown/test/runtests.jl              | 22 ++++++++++++++
 stdlib/REPL/src/docview.jl                    | 29 ++++++++++++-------
 stdlib/REPL/test/repl.jl                      | 20 +++++++++++++
 test/docs.jl                                  | 22 +++++++++++---
 9 files changed, 127 insertions(+), 15 deletions(-)

diff --git a/NEWS.md b/NEWS.md
index f29ba016966e1..50b260e9d3d3c 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -12,6 +12,14 @@ Language changes
   where it used to be incorrectly allowed. This is because `NTuple` refers only to homogeneous
   tuples (this meaning has not changed) ([#34272]).
 
+* In docstrings, a level-1 markdown header "Extended help" is now
+  interpreted as a marker dividing "brief help" from "extended help."
+  The REPL help mode only shows the brief help (the content before the
+  "Extended help" header) by default; prepend the expression with '?'
+  (in addition to the one that enters the help mode) to see the full
+  docstring. ([#25903])
+
+
 Multi-threading changes
 -----------------------
 
diff --git a/doc/src/manual/documentation.md b/doc/src/manual/documentation.md
index e191e89ef7b6e..a15e852b054a0 100644
--- a/doc/src/manual/documentation.md
+++ b/doc/src/manual/documentation.md
@@ -197,6 +197,10 @@ As in the example above, we recommend following some simple conventions when wri
    rather than users, explaining e.g. which functions should be overridden and which
    functions automatically use appropriate fallbacks. Such details are best kept separate
    from the main description of the function's behavior.
+5. For long docstrings, consider splitting the documentation with an
+   `# Extended help` header. The typical help-mode will show only the
+   material above the header; you can access the full help by adding a '?'
+   at the beginning of the expression (i.e., "??foo" rather than "?foo").
 
 ## Accessing Documentation
 
diff --git a/stdlib/Markdown/src/Markdown.jl b/stdlib/Markdown/src/Markdown.jl
index 35b3851febd0e..6220907649b3c 100644
--- a/stdlib/Markdown/src/Markdown.jl
+++ b/stdlib/Markdown/src/Markdown.jl
@@ -60,4 +60,30 @@ import Base.Docs: catdoc
 
 catdoc(md::MD...) = MD(md...)
 
+function trimdocs(md::MD, brief::Bool)
+    brief || return md
+    md, trimmed = _trimdocs(md, brief)
+    if trimmed
+        push!(md.content, Message("Extended help is available with `??`", (color=Base.info_color(), bold=true)))
+    end
+    return md
+end
+
+function _trimdocs(md::MD, brief::Bool)
+    content, trimmed = [], false
+    for c in md.content
+        if isa(c, Header{1}) && isa(c.text, AbstractArray) &&
+                                lowercase(c.text[1]) == "extended help"
+            trimmed = true
+            break
+        end
+        c, trm = _trimdocs(c, brief)
+        trimmed |= trm
+        push!(content, c)
+    end
+    return MD(content, md.meta), trimmed
+end
+
+_trimdocs(md, brief::Bool) = md, false
+
 end
diff --git a/stdlib/Markdown/src/parse/parse.jl b/stdlib/Markdown/src/parse/parse.jl
index 0da5f98a9a5ee..425b546383bc2 100644
--- a/stdlib/Markdown/src/parse/parse.jl
+++ b/stdlib/Markdown/src/parse/parse.jl
@@ -8,6 +8,13 @@ mutable struct MD
         new(content, meta)
 end
 
+# For direct messages to the terminal
+struct Message
+    msg    # AbstractString
+    fmt    # keywords to `printstyled`
+end
+Message(msg) = Message(msg, ())
+
 MD(xs...) = MD(vcat(xs...))
 
 function MD(cfg::Config, xs...)
diff --git a/stdlib/Markdown/src/render/terminal/render.jl b/stdlib/Markdown/src/render/terminal/render.jl
index 010f226a6bcd8..b68a74388f286 100644
--- a/stdlib/Markdown/src/render/terminal/render.jl
+++ b/stdlib/Markdown/src/render/terminal/render.jl
@@ -116,6 +116,10 @@ function term(io::IO, br::HorizontalRule, columns)
    print(io, ' '^margin, '─'^(columns - 2margin))
 end
 
+function term(io::IO, msg::Message, columns)
+    printstyled(io, msg.msg; msg.fmt...)
+end
+
 term(io::IO, x, _) = show(io, MIME"text/plain"(), x)
 
 # Inline Content
diff --git a/stdlib/Markdown/test/runtests.jl b/stdlib/Markdown/test/runtests.jl
index 47c977682ffc8..0fc0dd90080ff 100644
--- a/stdlib/Markdown/test/runtests.jl
+++ b/stdlib/Markdown/test/runtests.jl
@@ -1131,3 +1131,25 @@ let m = Markdown.parse("---"), io = IOBuffer()
     show(io, "text/latex", m)
     @test String(take!(io)) == "\\rule{\\textwidth}{1pt}\n"
 end
+
+# Brief and extended docs (issue #25930)
+let text =
+        """
+            brief_extended()
+
+        Short docs
+
+        # Extended help
+
+        Long docs
+        """,
+        md = Markdown.parse(text)
+    @test md == Markdown.trimdocs(md, false)
+    @test !isa(md.content[end], Markdown.Message)
+    mdbrief = Markdown.trimdocs(md, true)
+    @test length(mdbrief.content) == 3
+    @test isa(mdbrief.content[1], Markdown.Code)
+    @test isa(mdbrief.content[2], Markdown.Paragraph)
+    @test isa(mdbrief.content[3], Markdown.Message)
+    @test occursin("??", mdbrief.content[3].msg)
+end
diff --git a/stdlib/REPL/src/docview.jl b/stdlib/REPL/src/docview.jl
index a96e42317b065..31c36443b4f72 100644
--- a/stdlib/REPL/src/docview.jl
+++ b/stdlib/REPL/src/docview.jl
@@ -21,6 +21,12 @@ helpmode(line::AbstractString) = helpmode(stdout, line)
 
 function _helpmode(io::IO, line::AbstractString)
     line = strip(line)
+    if startswith(line, '?')
+        line = line[2:end]
+        brief = false
+    else
+        brief = true
+    end
     x = Meta.parse(line, raise = false, depwarn = false)
     expr =
         if haskey(keywords, Symbol(line)) || isexpr(x, :error) || isexpr(x, :invalid)
@@ -37,7 +43,7 @@ function _helpmode(io::IO, line::AbstractString)
         end
     # the following must call repl(io, expr) via the @repl macro
     # so that the resulting expressions are evaluated in the Base.Docs namespace
-    :($REPL.@repl $io $expr)
+    :($REPL.@repl $io $expr $brief)
 end
 _helpmode(line::AbstractString) = _helpmode(stdout, line)
 
@@ -273,10 +279,10 @@ function repl_latex(io::IO, s::String)
 end
 repl_latex(s::String) = repl_latex(stdout, s)
 
-macro repl(ex) repl(ex) end
-macro repl(io, ex) repl(io, ex) end
+macro repl(ex, brief=false) repl(ex; brief=brief) end
+macro repl(io, ex, brief) repl(io, ex; brief=brief) end
 
-function repl(io::IO, s::Symbol)
+function repl(io::IO, s::Symbol; brief::Bool=true)
     str = string(s)
     quote
         repl_latex($io, $str)
@@ -284,18 +290,18 @@ function repl(io::IO, s::Symbol)
         $(if !isdefined(Main, s) && !haskey(keywords, s)
                :(repl_corrections($io, $str))
           end)
-        $(_repl(s))
+        $(_repl(s, brief))
     end
 end
 isregex(x) = isexpr(x, :macrocall, 3) && x.args[1] === Symbol("@r_str") && !isempty(x.args[3])
-repl(io::IO, ex::Expr) = isregex(ex) ? :(apropos($io, $ex)) : _repl(ex)
-repl(io::IO, str::AbstractString) = :(apropos($io, $str))
-repl(io::IO, other) = esc(:(@doc $other))
+repl(io::IO, ex::Expr; brief::Bool=true) = isregex(ex) ? :(apropos($io, $ex)) : _repl(ex, brief)
+repl(io::IO, str::AbstractString; brief::Bool=true) = :(apropos($io, $str))
+repl(io::IO, other; brief::Bool=true) = esc(:(@doc $other))
 #repl(io::IO, other) = lookup_doc(other) # TODO
 
-repl(x) = repl(stdout, x)
+repl(x; brief=true) = repl(stdout, x; brief=brief)
 
-function _repl(x)
+function _repl(x, brief=true)
     if isexpr(x, :call)
         # determine the types of the values
         kwargs = nothing
@@ -349,7 +355,7 @@ function _repl(x)
     end
     #docs = lookup_doc(x) # TODO
     docs = esc(:(@doc $x))
-    if isfield(x)
+    docs = if isfield(x)
         quote
             if isa($(esc(x.args[1])), DataType)
                 fielddoc($(esc(x.args[1])), $(esc(x.args[2])))
@@ -360,6 +366,7 @@ function _repl(x)
     else
         docs
     end
+    :(Markdown.trimdocs($docs, $brief))
 end
 
 """
diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl
index b4c08dc821317..783fcf304ff84 100644
--- a/stdlib/REPL/test/repl.jl
+++ b/stdlib/REPL/test/repl.jl
@@ -1040,6 +1040,26 @@ for line in ["′", "abstract", "type", "|=", ".="]
         sprint(show, Base.eval(REPL._helpmode(IOBuffer(), line))::Union{Markdown.MD,Nothing}))
 end
 
+# Issue #25930
+module BriefExtended
+"""
+    f()
+
+Short docs
+
+# Extended help
+
+Long docs
+"""
+f() = nothing
+end # module BriefExtended
+buf = IOBuffer()
+md = Base.eval(REPL._helpmode(buf, "$(@__MODULE__).BriefExtended.f"))
+@test length(md.content) == 2 && isa(md.content[2], Markdown.Message)
+buf = IOBuffer()
+md = Base.eval(REPL._helpmode(buf, "?$(@__MODULE__).BriefExtended.f"))
+@test length(md.content) == 1 && length(md.content[1].content[1].content) == 4
+
 # PR #27562
 fake_repl() do stdin_write, stdout_read, repl
     repltask = @async begin
diff --git a/test/docs.jl b/test/docs.jl
index 25d674e9ca082..bbc4358ffa895 100644
--- a/test/docs.jl
+++ b/test/docs.jl
@@ -1011,28 +1011,42 @@ dynamic_test.x = "test 2"
 @test @doc(dynamic_test) == "test 2 Union{}"
 @test @doc(dynamic_test(::String)) == "test 2 Tuple{String}"
 
-let dt1 = _repl(:(dynamic_test(1.0)))
+# For testing purposes, strip off the `trimdocs(expr)` wrapper
+function striptrimdocs(expr)
+    if Meta.isexpr(expr, :call)
+        fex = expr.args[1]
+        if Meta.isexpr(fex, :.) && fex.args[1] == :Markdown
+            fmex = fex.args[2]
+            if isa(fmex, QuoteNode) && fmex.value == :trimdocs
+                expr = expr.args[2]
+            end
+        end
+    end
+    return expr
+end
+
+let dt1 = striptrimdocs(_repl(:(dynamic_test(1.0))))
     @test dt1 isa Expr
     @test dt1.args[1] isa Expr
     @test dt1.args[1].head === :macrocall
     @test dt1.args[1].args[1] == Symbol("@doc")
     @test dt1.args[1].args[3] == :(dynamic_test(::typeof(1.0)))
 end
-let dt2 = _repl(:(dynamic_test(::String)))
+let dt2 = striptrimdocs(_repl(:(dynamic_test(::String))))
     @test dt2 isa Expr
     @test dt2.args[1] isa Expr
     @test dt2.args[1].head === :macrocall
     @test dt2.args[1].args[1] == Symbol("@doc")
     @test dt2.args[1].args[3] == :(dynamic_test(::String))
 end
-let dt3 = _repl(:(dynamic_test(a)))
+let dt3 = striptrimdocs(_repl(:(dynamic_test(a))))
     @test dt3 isa Expr
     @test dt3.args[1] isa Expr
     @test dt3.args[1].head === :macrocall
     @test dt3.args[1].args[1] == Symbol("@doc")
     @test dt3.args[1].args[3].args[2].head == :(::) # can't test equality due to line numbers
 end
-let dt4 = _repl(:(dynamic_test(1.0,u=2.0)))
+let dt4 = striptrimdocs(_repl(:(dynamic_test(1.0,u=2.0))))
     @test dt4 isa Expr
     @test dt4.args[1] isa Expr
     @test dt4.args[1].head === :macrocall

From 3c6f01f5264a34c9e3f546ba649ee348c64a90d4 Mon Sep 17 00:00:00 2001
From: Tim Holy <tim.holy@gmail.com>
Date: Wed, 22 Jan 2020 03:24:19 -0600
Subject: [PATCH 2/3] Move extended docs handling to REPL

---
 stdlib/Markdown/src/Markdown.jl               | 26 ------------
 stdlib/Markdown/src/parse/parse.jl            |  7 ----
 stdlib/Markdown/src/render/terminal/render.jl |  4 --
 stdlib/Markdown/test/runtests.jl              | 22 ----------
 stdlib/REPL/src/docview.jl                    | 40 ++++++++++++++++++-
 stdlib/REPL/test/repl.jl                      | 25 +++++++++++-
 test/docs.jl                                  |  2 +-
 7 files changed, 64 insertions(+), 62 deletions(-)

diff --git a/stdlib/Markdown/src/Markdown.jl b/stdlib/Markdown/src/Markdown.jl
index 6220907649b3c..35b3851febd0e 100644
--- a/stdlib/Markdown/src/Markdown.jl
+++ b/stdlib/Markdown/src/Markdown.jl
@@ -60,30 +60,4 @@ import Base.Docs: catdoc
 
 catdoc(md::MD...) = MD(md...)
 
-function trimdocs(md::MD, brief::Bool)
-    brief || return md
-    md, trimmed = _trimdocs(md, brief)
-    if trimmed
-        push!(md.content, Message("Extended help is available with `??`", (color=Base.info_color(), bold=true)))
-    end
-    return md
-end
-
-function _trimdocs(md::MD, brief::Bool)
-    content, trimmed = [], false
-    for c in md.content
-        if isa(c, Header{1}) && isa(c.text, AbstractArray) &&
-                                lowercase(c.text[1]) == "extended help"
-            trimmed = true
-            break
-        end
-        c, trm = _trimdocs(c, brief)
-        trimmed |= trm
-        push!(content, c)
-    end
-    return MD(content, md.meta), trimmed
-end
-
-_trimdocs(md, brief::Bool) = md, false
-
 end
diff --git a/stdlib/Markdown/src/parse/parse.jl b/stdlib/Markdown/src/parse/parse.jl
index 425b546383bc2..0da5f98a9a5ee 100644
--- a/stdlib/Markdown/src/parse/parse.jl
+++ b/stdlib/Markdown/src/parse/parse.jl
@@ -8,13 +8,6 @@ mutable struct MD
         new(content, meta)
 end
 
-# For direct messages to the terminal
-struct Message
-    msg    # AbstractString
-    fmt    # keywords to `printstyled`
-end
-Message(msg) = Message(msg, ())
-
 MD(xs...) = MD(vcat(xs...))
 
 function MD(cfg::Config, xs...)
diff --git a/stdlib/Markdown/src/render/terminal/render.jl b/stdlib/Markdown/src/render/terminal/render.jl
index b68a74388f286..010f226a6bcd8 100644
--- a/stdlib/Markdown/src/render/terminal/render.jl
+++ b/stdlib/Markdown/src/render/terminal/render.jl
@@ -116,10 +116,6 @@ function term(io::IO, br::HorizontalRule, columns)
    print(io, ' '^margin, '─'^(columns - 2margin))
 end
 
-function term(io::IO, msg::Message, columns)
-    printstyled(io, msg.msg; msg.fmt...)
-end
-
 term(io::IO, x, _) = show(io, MIME"text/plain"(), x)
 
 # Inline Content
diff --git a/stdlib/Markdown/test/runtests.jl b/stdlib/Markdown/test/runtests.jl
index 0fc0dd90080ff..47c977682ffc8 100644
--- a/stdlib/Markdown/test/runtests.jl
+++ b/stdlib/Markdown/test/runtests.jl
@@ -1131,25 +1131,3 @@ let m = Markdown.parse("---"), io = IOBuffer()
     show(io, "text/latex", m)
     @test String(take!(io)) == "\\rule{\\textwidth}{1pt}\n"
 end
-
-# Brief and extended docs (issue #25930)
-let text =
-        """
-            brief_extended()
-
-        Short docs
-
-        # Extended help
-
-        Long docs
-        """,
-        md = Markdown.parse(text)
-    @test md == Markdown.trimdocs(md, false)
-    @test !isa(md.content[end], Markdown.Message)
-    mdbrief = Markdown.trimdocs(md, true)
-    @test length(mdbrief.content) == 3
-    @test isa(mdbrief.content[1], Markdown.Code)
-    @test isa(mdbrief.content[2], Markdown.Paragraph)
-    @test isa(mdbrief.content[3], Markdown.Message)
-    @test occursin("??", mdbrief.content[3].msg)
-end
diff --git a/stdlib/REPL/src/docview.jl b/stdlib/REPL/src/docview.jl
index 31c36443b4f72..47d61afeaa16e 100644
--- a/stdlib/REPL/src/docview.jl
+++ b/stdlib/REPL/src/docview.jl
@@ -79,6 +79,44 @@ function parsedoc(d::DocStr)
     d.object
 end
 
+## Trimming long help ("# Extended help")
+
+struct Message  # For direct messages to the terminal
+    msg    # AbstractString
+    fmt    # keywords to `printstyled`
+end
+Message(msg) = Message(msg, ())
+
+function Markdown.term(io::IO, msg::Message, columns)
+    printstyled(io, msg.msg; msg.fmt...)
+end
+
+function trimdocs(md::Markdown.MD, brief::Bool)
+    brief || return md
+    md, trimmed = _trimdocs(md, brief)
+    if trimmed
+        push!(md.content, Message("Extended help is available with `??`", (color=Base.info_color(), bold=true)))
+    end
+    return md
+end
+
+function _trimdocs(md::Markdown.MD, brief::Bool)
+    content, trimmed = [], false
+    for c in md.content
+        if isa(c, Markdown.Header{1}) && isa(c.text, AbstractArray) &&
+                                         lowercase(c.text[1]) == "extended help"
+            trimmed = true
+            break
+        end
+        c, trm = _trimdocs(c, brief)
+        trimmed |= trm
+        push!(content, c)
+    end
+    return Markdown.MD(content, md.meta), trimmed
+end
+
+_trimdocs(md, brief::Bool) = md, false
+
 """
     Docs.doc(binding, sig)
 
@@ -366,7 +404,7 @@ function _repl(x, brief=true)
     else
         docs
     end
-    :(Markdown.trimdocs($docs, $brief))
+    :(REPL.trimdocs($docs, $brief))
 end
 
 """
diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl
index 783fcf304ff84..a2b06eaab701f 100644
--- a/stdlib/REPL/test/repl.jl
+++ b/stdlib/REPL/test/repl.jl
@@ -1041,6 +1041,29 @@ for line in ["′", "abstract", "type", "|=", ".="]
 end
 
 # Issue #25930
+
+# Brief and extended docs (issue #25930)
+let text =
+        """
+            brief_extended()
+
+        Short docs
+
+        # Extended help
+
+        Long docs
+        """,
+    md = Markdown.parse(text)
+    @test md == REPL.trimdocs(md, false)
+    @test !isa(md.content[end], REPL.Message)
+    mdbrief = REPL.trimdocs(md, true)
+    @test length(mdbrief.content) == 3
+    @test isa(mdbrief.content[1], Markdown.Code)
+    @test isa(mdbrief.content[2], Markdown.Paragraph)
+    @test isa(mdbrief.content[3], REPL.Message)
+    @test occursin("??", mdbrief.content[3].msg)
+end
+
 module BriefExtended
 """
     f()
@@ -1055,7 +1078,7 @@ f() = nothing
 end # module BriefExtended
 buf = IOBuffer()
 md = Base.eval(REPL._helpmode(buf, "$(@__MODULE__).BriefExtended.f"))
-@test length(md.content) == 2 && isa(md.content[2], Markdown.Message)
+@test length(md.content) == 2 && isa(md.content[2], REPL.Message)
 buf = IOBuffer()
 md = Base.eval(REPL._helpmode(buf, "?$(@__MODULE__).BriefExtended.f"))
 @test length(md.content) == 1 && length(md.content[1].content[1].content) == 4
diff --git a/test/docs.jl b/test/docs.jl
index bbc4358ffa895..510152f15be44 100644
--- a/test/docs.jl
+++ b/test/docs.jl
@@ -1015,7 +1015,7 @@ dynamic_test.x = "test 2"
 function striptrimdocs(expr)
     if Meta.isexpr(expr, :call)
         fex = expr.args[1]
-        if Meta.isexpr(fex, :.) && fex.args[1] == :Markdown
+        if Meta.isexpr(fex, :.) && fex.args[1] == :REPL
             fmex = fex.args[2]
             if isa(fmex, QuoteNode) && fmex.value == :trimdocs
                 expr = expr.args[2]

From 30c7ca0d860afc208d4357470ea88217ce16d9a2 Mon Sep 17 00:00:00 2001
From: Tim Holy <tim.holy@gmail.com>
Date: Fri, 24 Jan 2020 02:55:02 -0600
Subject: [PATCH 3/3] Address review suggestions

---
 stdlib/REPL/src/docview.jl | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/stdlib/REPL/src/docview.jl b/stdlib/REPL/src/docview.jl
index 47d61afeaa16e..ffbb86a1aa4a2 100644
--- a/stdlib/REPL/src/docview.jl
+++ b/stdlib/REPL/src/docview.jl
@@ -19,12 +19,16 @@ using InteractiveUtils: subtypes
 helpmode(io::IO, line::AbstractString) = :($REPL.insert_hlines($io, $(REPL._helpmode(io, line))))
 helpmode(line::AbstractString) = helpmode(stdout, line)
 
+const extended_help_on = Ref{Any}(nothing)
+
 function _helpmode(io::IO, line::AbstractString)
     line = strip(line)
     if startswith(line, '?')
         line = line[2:end]
+        extended_help_on[] = line
         brief = false
     else
+        extended_help_on[] = nothing
         brief = true
     end
     x = Meta.parse(line, raise = false, depwarn = false)
@@ -95,7 +99,9 @@ function trimdocs(md::Markdown.MD, brief::Bool)
     brief || return md
     md, trimmed = _trimdocs(md, brief)
     if trimmed
-        push!(md.content, Message("Extended help is available with `??`", (color=Base.info_color(), bold=true)))
+        line = extended_help_on[]
+        line = isa(line, AbstractString) ? line : ""
+        push!(md.content, Message("Extended help is available with `??$line`", (color=Base.info_color(), bold=true)))
     end
     return md
 end
@@ -104,7 +110,9 @@ function _trimdocs(md::Markdown.MD, brief::Bool)
     content, trimmed = [], false
     for c in md.content
         if isa(c, Markdown.Header{1}) && isa(c.text, AbstractArray) &&
-                                         lowercase(c.text[1]) == "extended help"
+                                         lowercase(c.text[1]) ∈ ("extended help",
+                                                                 "extended documentation",
+                                                                 "extended docs")
             trimmed = true
             break
         end