Add chomp option to readline(s)

NEWS.md

@@ -33,6 +33,9 @@ Breaking changes
 
 This section lists changes that do not have deprecation warnings.
 
+  * `readline`, `readlines` and `eachline` return lines without line ends by default.
+     You can use `readline(s, false`) to get the old behavior and include EOL character(s). ([#19944]).
+
   * `String`s no longer have a `.data` field (as part of a significant performance
     improvement). Use `Vector{UInt8}(str)` to access a string as a byte array.
     However, allocating the `Vector` object has overhead. You can also use

base/LineEdit.jl

@@ -211,7 +211,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf
     seek(buf, 0)
     moreinput = true # add a blank line if there is a trailing newline on the last line
     while moreinput
-        l = readline(buf)
+        l = readline(buf, false)
         moreinput = endswith(l, "\n")
         # We need to deal with on-screen characters, so use strwidth to compute occupied columns
         llength = strwidth(l)
@@ -549,7 +549,7 @@ end
 function edit_kill_line(s::MIState)
     buf = buffer(s)
     pos = position(buf)
-    killbuf = readline(buf)
+    killbuf = readline(buf, false)
     if length(killbuf) > 1 && killbuf[end] == '\n'
         killbuf = killbuf[1:end-1]
         char_move_left(buf)

base/REPL.jl

@@ -205,7 +205,7 @@ function run_frontend(repl::BasicREPL, backend::REPLBackendRef)
         interrupted = false
         while true
             try
-                line *= readline(repl.terminal)
+                line *= readline(repl.terminal, false)
             catch e
                 if isa(e,InterruptException)
                     try # raise the debugger if present
@@ -337,7 +337,7 @@ An editor may have converted tabs to spaces at line """
 
 function hist_getline(file)
     while !eof(file)
-        line = readline(file)
+        line = readline(file, false)
         isempty(line) && return line
         line[1] in "\r\n" || return line
     end
@@ -995,7 +995,7 @@ function run_frontend(repl::StreamREPL, backend::REPLBackendRef)
         if have_color
             print(repl.stream, input_color(repl))
         end
-        line = readline(repl.stream)
+        line = readline(repl.stream, false)
         if !isempty(line)
             ast = Base.parse_input_line(line)
             if have_color

base/client.jl

@@ -216,7 +216,7 @@ parse_input_line(s::AbstractString) = parse_input_line(String(s))
 function parse_input_line(io::IO)
     s = ""
     while !eof(io)
-        s = s*readline(io)
+        s = s*readline(io, false)
         e = parse_input_line(s)
         if !(isa(e,Expr) && e.head === :incomplete)
             return e

base/io.jl

@@ -167,24 +167,53 @@ The text is assumed to be encoded in UTF-8.
 readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), filename)
 
 """
-    readline(stream::IO=STDIN)
-    readline(filename::AbstractString)
+    readline()
+    readline(stream, chomp::Bool=true)
+    readline(filename::AbstractString, chomp::Bool=true)
 
-Read a single line of text, including a trailing newline character (if one is reached before
-the end of the input), from the given I/O stream or file (defaults to `STDIN`).
-When reading from a file, the text is assumed to be encoded in UTF-8.
+Read a single line of text from the given I/O stream or file (defaults to `STDIN`).
+Lines in the input can end in `'\\n'` or `"\\r\\n"`. When reading from a file, the text is
+assumed to be encoded in UTF-8.
+
+If `chomp=false` trailing newline character(s) will be included in the output
+(if reached before the end of the input); otherwise newline characters(s)
+are stripped from result.
 """
-readline(filename::AbstractString) = open(readline, filename)
+function readline(filename::AbstractString, chomp::Bool=true)
+    open(filename) do f
+          readline(f, chomp)
+    end
+end
+readline() = readline(STDIN, false)
+
+function readline(s::IO, chomp::Bool=true)
+    line = readuntil(s, 0x0a)
+    i = length(line)
+    if !chomp || i == 0 || line[i] != 0x0a
+        return String(line)
+    elseif i < 2 || line[i-1] != 0x0d
+        return String(resize!(line,i-1))
+    else
+        return String(resize!(line,i-2))
+    end
+end
 
 """
-    readlines(stream::IO)
-    readlines(filename::AbstractString)
+    readlines(stream::IO, chomp::Bool=true)
+    readlines(filename::AbstractString, chomp::Bool=true)
 
 Read all lines of an I/O stream or a file as a vector of strings.
+Lines in the input can end in `'\\n'` or `"\\r\\n"`.
 The text is assumed to be encoded in UTF-8.
-"""
-readlines(filename::AbstractString) = open(readlines, filename)
 
+If `chomp=false` trailing newline character(s) will be included in the output;
+otherwise newline characters(s) are stripped from result.
+"""
+function readlines(filename::AbstractString, chomp::Bool=true)
+    open(filename) do f
+        readlines(f, chomp)
+    end
+end
 
 ## byte-order mark, ntoh & hton ##
 
@@ -447,9 +476,6 @@ function readuntil(s::IO, t::AbstractString)
     return String(take!(out))
 end
 
-readline() = readline(STDIN)
-readline(s::IO) = readuntil(s, '\n')
-
 """
     readchomp(x)
 
@@ -512,22 +538,28 @@ readstring(filename::AbstractString) = open(readstring, filename)
 
 type EachLine
     stream::IO
+    chomp::Bool
     ondone::Function
-    EachLine(stream) = EachLine(stream, ()->nothing)
-    EachLine(stream, ondone) = new(stream, ondone)
+    EachLine(stream, chomp) = EachLine(stream, chomp, ()->nothing)
+    EachLine(stream, chomp, ondone) = new(stream, chomp, ondone)
 end
 
 """
-    eachline(stream::IO)
-    eachline(filename::AbstractString)
+    eachline(stream::IO,  chomp::Bool=true)
+    eachline(filename::AbstractString,  chomp::Bool=true)
 
 Create an iterable object that will yield each line from an I/O stream or a file.
+Lines in the input can end in `'\\n'` or `"\\r\\n"`.
 The text is assumed to be encoded in UTF-8.
+
+If `chomp=false` trailing newline character(s) will be included in the output;
+otherwise newline characters(s) are stripped from result.
 """
-eachline(stream::IO) = EachLine(stream)
-function eachline(filename::AbstractString)
+eachline(stream::IO, chomp::Bool=true) = EachLine(stream, chomp)
+
+function eachline(filename::AbstractString, chomp::Bool=true)
     s = open(filename)
-    EachLine(s, ()->close(s))
+    EachLine(s, chomp, ()->close(s))
 end
 
 start(itr::EachLine) = nothing
@@ -538,10 +570,11 @@ function done(itr::EachLine, nada)
     itr.ondone()
     true
 end
-next(itr::EachLine, nada) = (readline(itr.stream), nothing)
+
+next(itr::EachLine, nada) = (readline(itr.stream, itr.chomp), nothing)
 eltype(::Type{EachLine}) = String
 
-readlines(s=STDIN) = collect(eachline(s))
+readlines(s::IO, chomp::Bool=true) = collect(eachline(s, chomp))
 
 iteratorsize(::Type{EachLine}) = SizeUnknown()
 

base/iostream.jl

@@ -222,11 +222,11 @@ take!(s::IOStream) =
     ccall(:jl_take_buffer, Vector{UInt8}, (Ptr{Void},), s.ios)
 
 function readuntil(s::IOStream, delim::UInt8)
-    ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8), s.ios, delim, 0)
+    ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, delim, 0, 0)
 end
 
-function readline(s::IOStream)
-    ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8), s.ios, '\n', 1)
+function readline(s::IOStream, chomp::Bool=true)
+    ccall(:jl_readuntil, Ref{String}, (Ptr{Void}, UInt8, UInt8, UInt8), s.ios, '\n', 1, chomp)
 end
 
 function readbytes_all!(s::IOStream, b::Array{UInt8}, nb)
@@ -320,7 +320,7 @@ function skipchars(s::IOStream, pred; linecomment::Char=Char(0xffffffff))
     ch = peekchar(s); status = Int(ch)
     while status >= 0 && (pred(ch) || ch == linecomment)
         if ch == linecomment
-            readline(s)
+            readline(s, false)
         else
             read(s, Char)  # advance one character
         end

base/libgit2/callbacks.jl

@@ -99,7 +99,7 @@ function authenticate_ssh(creds::SSHCredentials, libgit2credptr::Ptr{Ptr{Void}},
         else
             # In encrypted private keys, the second line is "Proc-Type: 4,ENCRYPTED"
             open(privatekey) do f
-                passphrase_required = (readline(f); chomp(readline(f)) == "Proc-Type: 4,ENCRYPTED")
+                passphrase_required = (readline(f, false); readline(f, true) == "Proc-Type: 4,ENCRYPTED")
             end
         end
 

base/libgit2/utils.jl

@@ -31,7 +31,7 @@ function prompt(msg::AbstractString; default::AbstractString="", password::Bool=
         Base.getpass(msg)
     else
         print(msg)
-        chomp(readline(STDIN))
+        readline(STDIN, true)
     end
     isempty(uinput) ? default : uinput
 end

base/markdown/Common/block.jl

@@ -61,7 +61,7 @@ function hashheader(stream::IO, md::MD)
             return false
 
         if c != '\n' # Empty header
-            h = readline(stream) |> strip
+            h = readline(stream, false) |> strip
             h = match(r"(.*?)( +#+)?$", h).captures[1]
             buffer = IOBuffer()
             print(buffer, h)
@@ -76,11 +76,11 @@ end
 function setextheader(stream::IO, md::MD)
     withstream(stream) do
         eatindent(stream) || return false
-        header = readline(stream) |> strip
+        header = readline(stream, false) |> strip
         header == "" && return false
 
         eatindent(stream) || return false
-        underline = readline(stream) |> strip
+        underline = readline(stream, false) |> strip
         length(underline) < 3 && return false
         u = underline[1]
         u in "-=" || return false
@@ -108,7 +108,7 @@ function indentcode(stream::IO, block::MD)
         buffer = IOBuffer()
         while !eof(stream)
             if startswith(stream, "    ") || startswith(stream, "\t")
-                write(buffer, readline(stream))
+                write(buffer, readline(stream, false))
             elseif blankline(stream)
                 write(buffer, '\n')
             else
@@ -139,10 +139,10 @@ function footnote(stream::IO, block::MD)
         else
             ref = match(regex, str).captures[1]
             buffer = IOBuffer()
-            write(buffer, readline(stream))
+            write(buffer, readline(stream, false))
             while !eof(stream)
                 if startswith(stream, "    ")
-                    write(buffer, readline(stream))
+                    write(buffer, readline(stream, false))
                 elseif blankline(stream)
                     write(buffer, '\n')
                 else
@@ -174,7 +174,7 @@ function blockquote(stream::IO, block::MD)
         empty = true
         while eatindent(stream) && startswith(stream, '>')
             startswith(stream, " ")
-            write(buffer, readline(stream))
+            write(buffer, readline(stream, false))
             empty = false
         end
         empty && return false
@@ -210,7 +210,7 @@ function admonition(stream::IO, block::MD)
         category, title =
             let untitled = r"^([a-z]+)$",          # !!! <CATEGORY_NAME>
                 titled   = r"^([a-z]+) \"(.*)\"$", # !!! <CATEGORY_NAME> "<TITLE>"
-                line     = strip(readline(stream))
+                line     = strip(readline(stream, false))
                 if ismatch(untitled, line)
                     m = match(untitled, line)
                     # When no title is provided we use CATEGORY_NAME, capitalising it.
@@ -229,7 +229,7 @@ function admonition(stream::IO, block::MD)
         buffer = IOBuffer()
         while !eof(stream)
             if startswith(stream, "    ")
-                write(buffer, readline(stream))
+                write(buffer, readline(stream, false))
             elseif blankline(stream)
                 write(buffer, '\n')
             else
@@ -305,7 +305,7 @@ function list(stream::IO, block::MD)
                 newline = false
                 if startswith(stream, " "^indent)
                     # Indented text that is part of the current list item.
-                    print(buffer, readline(stream))
+                    print(buffer, readline(stream, false))
                 else
                     matched = startswith(stream, regex)
                     if isempty(matched)
@@ -316,7 +316,7 @@ function list(stream::IO, block::MD)
                         # Start of a new list item.
                         count += 1
                         count > 1 && pushitem!(list, buffer)
-                        print(buffer, readline(stream))
+                        print(buffer, readline(stream, false))
                     end
                 end
             end

base/markdown/GitHub/GitHub.jl

@@ -8,7 +8,7 @@ function fencedcode(stream::IO, block::MD)
         startswith(stream, "~~~", padding = true) || startswith(stream, "```", padding = true) || return false
         skip(stream, -1)
         ch = read(stream, Char)
-        trailing = strip(readline(stream))
+        trailing = strip(readline(stream, false))
         flavor = lstrip(trailing, ch)
         n = 3 + length(trailing) - length(flavor)
 
@@ -30,7 +30,7 @@ function fencedcode(stream::IO, block::MD)
                     seek(stream, line_start)
                 end
             end
-            write(buffer, readline(stream))
+            write(buffer, readline(stream, false))
         end
         return false
     end

base/markdown/GitHub/table.jl

@@ -7,7 +7,7 @@ end
 
 function parserow(stream::IO)
     withstream(stream) do
-        line = readline(stream) |> chomp
+        line = readline(stream, true)
         row = split(line, r"(?<!\\)\|")
         length(row) == 1 && return
         row[1] == "" && shift!(row)

base/markdown/parse/util.jl

@@ -46,7 +46,7 @@ function linecontains(io::IO, chars; allow_whitespace = true,
                                      eat = true,
                                      allowempty = false)
     start = position(io)
-    l = readline(io) |> chomp
+    l = readline(io, true)
     length(l) == 0 && return allowempty
 
     result = allowempty
@@ -99,7 +99,7 @@ function startswith(stream::IO, r::Regex; eat = true, padding = false)
     @assert Base.startswith(r.pattern, "^")
     start = position(stream)
     padding && skipwhitespace(stream)
-    line = chomp(readline(stream))
+    line = readline(stream, true)
     seek(stream, start)
     m = match(r, line)
     m === nothing && return ""

base/multi.jl

@@ -1624,7 +1624,7 @@ end
 
 function redirect_worker_output(ident, stream)
     @schedule while !eof(stream)
-        line = readline(stream)
+        line = readline(stream, false)
         if startswith(line, "\tFrom worker ")
             # STDOUT's of "additional" workers started from an initial worker on a host are not available
             # on the master directly - they are routed via the initial worker's STDOUT.
@@ -1642,7 +1642,7 @@ end
 # setup a all-to-all network.
 function read_worker_host_port(io::IO)
     while true
-        conninfo = readline(io)
+        conninfo = readline(io, false)
         bind_addr, port = parse_connection_info(conninfo)
         if bind_addr != ""
             return bind_addr, port

base/pkg/dir.jl

@@ -67,7 +67,7 @@ end
 
 function getmetabranch()
     try
-        chomp(readline(joinpath(path(),"META_BRANCH")))
+        readline(joinpath(path(),"META_BRANCH"), true)
     catch err
         META_BRANCH
     end