Merge pull request #33979 from JuliaLang/backports-release-1.3

Backports for release v1.3.1

Make.inc

@@ -929,6 +929,20 @@ else
 PCRE_CONFIG := $(build_depsbindir)/pcre2-config
 endif
 
+ifeq ($(USE_SYSTEM_PATCHELF), 1)
+PATCHELF := patchelf
+else
+PATCHELF := $(build_depsbindir)/patchelf
+endif
+
+# On aarch64 and powerpc64le, we assume the page size is 64K.  Our binutils linkers
+# and such already assume this, but `patchelf` seems to be behind the times.  We
+# explicitly tell it to use this large page size so that when we rewrite rpaths and
+# such, we don't accidentally create incorrectly-aligned sections in our ELF files.
+ifneq (,$(filter $(ARCH),aarch64 powerpc64le))
+PATCHELF += --page-size=65536
+endif
+
 # Use ILP64 BLAS interface when building openblas from source on 64-bit architectures
 ifeq ($(BINARY), 64)
 ifeq ($(USE_SYSTEM_BLAS), 1)

Makefile

@@ -269,7 +269,7 @@ endef
 ifeq (,$(findstring $(OS),FreeBSD WINNT))
 julia-base: $(build_libdir)/libgfortran*.$(SHLIB_EXT)*
 $(build_libdir)/libgfortran*.$(SHLIB_EXT)*: | $(build_libdir) julia-deps
-	-$(CUSTOM_LD_LIBRARY_PATH) PATH="$(PATH):$(build_depsbindir)" $(JULIAHOME)/contrib/fixup-libgfortran.sh --verbose $(build_libdir)
+	-$(CUSTOM_LD_LIBRARY_PATH) PATH="$(PATH):$(build_depsbindir)" PATCHELF="$(PATCHELF)" $(JULIAHOME)/contrib/fixup-libgfortran.sh --verbose $(build_libdir)
 JL_PRIVATE_LIBS-0 += libgfortran libgcc_s libquadmath
 endif
 
@@ -374,7 +374,6 @@ endif
 	mkdir -p $(DESTDIR)$(datarootdir)/icons/hicolor/scalable/apps/
 	$(INSTALL_F) $(JULIAHOME)/contrib/julia.svg $(DESTDIR)$(datarootdir)/icons/hicolor/scalable/apps/
 	-touch -c $(DESTDIR)$(datarootdir)/icons/hicolor/
-	-gtk-update-icon-cache --ignore-theme-index $(DESTDIR)$(datarootdir)/icons/hicolor/
 	mkdir -p $(DESTDIR)$(datarootdir)/applications/
 	$(INSTALL_F) $(JULIAHOME)/contrib/julia.desktop $(DESTDIR)$(datarootdir)/applications/
 	# Install appdata file
@@ -392,7 +391,7 @@ ifneq ($(DARWIN_FRAMEWORK),1)
 endif
 else ifneq (,$(findstring $(OS),Linux FreeBSD))
 	for j in $(JL_TARGETS) ; do \
-		patchelf --set-rpath '$$ORIGIN/$(private_libdir_rel):$$ORIGIN/$(libdir_rel)' $(DESTDIR)$(bindir)/$$j; \
+		$(PATCHELF) --set-rpath '$$ORIGIN/$(private_libdir_rel):$$ORIGIN/$(libdir_rel)' $(DESTDIR)$(bindir)/$$j; \
 	done
 endif
 
@@ -412,15 +411,15 @@ endif
 endif
 	# On FreeBSD, remove the build's libdir from each library's RPATH
 ifeq ($(OS),FreeBSD)
-	$(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(libdir) $(build_libdir)
-	$(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(private_libdir) $(build_libdir)
-	$(JULIAHOME)/contrib/fixup-rpath.sh $(build_depsbindir)/patchelf $(DESTDIR)$(bindir) $(build_libdir)
+	$(JULIAHOME)/contrib/fixup-rpath.sh "$(PATCHELF)" $(DESTDIR)$(libdir) $(build_libdir)
+	$(JULIAHOME)/contrib/fixup-rpath.sh "$(PATCHELF)" $(DESTDIR)$(private_libdir) $(build_libdir)
+	$(JULIAHOME)/contrib/fixup-rpath.sh "$(PATCHELF)" $(DESTDIR)$(bindir) $(build_libdir)
 	# Set libgfortran's RPATH to ORIGIN instead of GCCPATH. It's only libgfortran that
 	# needs to be fixed here, as libgcc_s and libquadmath don't have RPATHs set. If we
 	# don't set libgfortran's RPATH, it won't be able to find its friends on systems
 	# that don't have the exact GCC port installed used for the build.
 	for lib in $(DESTDIR)$(private_libdir)/libgfortran*$(SHLIB_EXT)*; do \
-		$(build_depsbindir)/patchelf --set-rpath '$$ORIGIN' $$lib; \
+		$(PATCHELF) --set-rpath '$$ORIGIN' $$lib; \
 	done
 endif
 

base/intfuncs.jl

@@ -68,7 +68,7 @@ function lcm(a::T, b::T) where T<:Integer
     if a == 0
         return a
     else
-        return checked_abs(a * div(b, gcd(b,a)))
+        return checked_abs(checked_mul(a, div(b, gcd(b,a))))
     end
 end
 

base/regex.jl

@@ -669,7 +669,7 @@ regex_opts_str(opts) = (isassigned(_regex_opts_str) ? _regex_opts_str[] : init_r
 # UInt32 to String mapping for some compile options
 const _regex_opts_str = Ref{ImmutableDict{UInt32,String}}()
 
-init_regex() = _regex_opts_str[] = foldl(0:15, init=ImmutableDict{UInt32,String}()) do d, o
+@noinline init_regex() = _regex_opts_str[] = foldl(0:15, init=ImmutableDict{UInt32,String}()) do d, o
     opt = UInt32(0)
     str = ""
     if o & 1 != 0

base/reshapedarray.jl

@@ -112,6 +112,7 @@ reshape(parent::AbstractArray, shp::Tuple{Union{Integer,OneTo}, Vararg{Union{Int
 reshape(parent::AbstractArray, dims::Dims)        = _reshape(parent, dims)
 
 # Allow missing dimensions with Colon():
+reshape(parent::AbstractVector, ::Colon) = parent
 reshape(parent::AbstractArray, dims::Int...) = reshape(parent, dims)
 reshape(parent::AbstractArray, dims::Union{Int,Colon}...) = reshape(parent, dims)
 reshape(parent::AbstractArray, dims::Tuple{Vararg{Union{Int,Colon}}}) = _reshape(parent, _reshape_uncolon(parent, dims))
@@ -220,6 +221,8 @@ dataids(A::ReshapedArray) = dataids(A.parent)
     d, r = divrem(ind, strds[1])
     (_ind2sub_rs(front(ax), tail(strds), r)..., d + first(ax[end]))
 end
+offset_if_vec(i::Integer, axs::Tuple{<:AbstractUnitRange}) = i + first(axs[1]) - 1
+offset_if_vec(i::Integer, axs::Tuple) = i
 
 @inline function getindex(A::ReshapedArrayLF, index::Int)
     @boundscheck checkbounds(A, index)
@@ -237,8 +240,9 @@ end
 end
 
 @inline function _unsafe_getindex(A::ReshapedArray{T,N}, indices::Vararg{Int,N}) where {T,N}
-    i = Base._sub2ind(size(A), indices...)
-    I = ind2sub_rs(axes(A.parent), A.mi, i)
+    axp = axes(A.parent)
+    i = offset_if_vec(Base._sub2ind(size(A), indices...), axp)
+    I = ind2sub_rs(axp, A.mi, i)
     _unsafe_getindex_rs(parent(A), I)
 end
 @inline _unsafe_getindex_rs(A, i::Integer) = (@inbounds ret = A[i]; ret)
@@ -260,7 +264,9 @@ end
 end
 
 @inline function _unsafe_setindex!(A::ReshapedArray{T,N}, val, indices::Vararg{Int,N}) where {T,N}
-    @inbounds parent(A)[ind2sub_rs(axes(A.parent), A.mi, Base._sub2ind(size(A), indices...))...] = val
+    axp = axes(A.parent)
+    i = offset_if_vec(Base._sub2ind(size(A), indices...), axp)
+    @inbounds parent(A)[ind2sub_rs(axes(A.parent), A.mi, i)...] = val
     val
 end
 

base/stream.jl

@@ -917,7 +917,7 @@ function readuntil(x::LibuvStream, c::UInt8; keep::Bool=false)
     return bytes
 end
 
-uv_write(s::LibuvStream, p::Vector{UInt8}) = uv_write(s, pointer(p), UInt(sizeof(p)))
+uv_write(s::LibuvStream, p::Vector{UInt8}) = GC.@preserve p uv_write(s, pointer(p), UInt(sizeof(p)))
 
 # caller must have acquired the iolock
 function uv_write(s::LibuvStream, p::Ptr{UInt8}, n::UInt)
@@ -1036,8 +1036,9 @@ function write(s::LibuvStream, b::UInt8)
     if buf !== nothing
         iolock_begin()
         if bytesavailable(buf) + 1 < buf.maxsize
+            n = write(buf, b)
             iolock_end()
-            return write(buf, b)
+            return n
         end
         iolock_end()
     end

base/strings/search.jl

@@ -34,8 +34,8 @@ function _search(a::Union{String,ByteArray}, b::Union{Int8,UInt8}, i::Integer =
         return i == n+1 ? 0 : throw(BoundsError(a, i))
     end
     p = pointer(a)
-    q = ccall(:memchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p+i-1, b, n-i+1)
-    q == C_NULL ? 0 : Int(q-p+1)
+    q = GC.@preserve a ccall(:memchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p+i-1, b, n-i+1)
+    return q == C_NULL ? 0 : Int(q-p+1)
 end
 
 function _search(a::ByteArray, b::AbstractChar, i::Integer = 1)
@@ -74,8 +74,8 @@ function _rsearch(a::Union{String,ByteArray}, b::Union{Int8,UInt8}, i::Integer =
         return i == n+1 ? 0 : throw(BoundsError(a, i))
     end
     p = pointer(a)
-    q = ccall(:memrchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p, b, i)
-    q == C_NULL ? 0 : Int(q-p+1)
+    q = GC.@preserve a ccall(:memrchr, Ptr{UInt8}, (Ptr{UInt8}, Int32, Csize_t), p, b, i)
+    return q == C_NULL ? 0 : Int(q-p+1)
 end
 
 function _rsearch(a::ByteArray, b::AbstractChar, i::Integer = length(a))

base/strings/string.jl

@@ -87,7 +87,8 @@ codeunit(s::String) = UInt8
 
 @inline function codeunit(s::String, i::Integer)
     @boundscheck checkbounds(s, i)
-    GC.@preserve s unsafe_load(pointer(s, i))
+    b = GC.@preserve s unsafe_load(pointer(s, i))
+    return b
 end
 
 ## comparison ##
@@ -112,7 +113,7 @@ typemin(::String) = typemin(String)
 
 ## thisind, nextind ##
 
-Base.@propagate_inbounds thisind(s::String, i::Int) = _thisind_str(s, i)
+@propagate_inbounds thisind(s::String, i::Int) = _thisind_str(s, i)
 
 # s should be String or SubString{String}
 @inline function _thisind_str(s, i::Int)
@@ -133,7 +134,7 @@ Base.@propagate_inbounds thisind(s::String, i::Int) = _thisind_str(s, i)
     return i
 end
 
-Base.@propagate_inbounds nextind(s::String, i::Int) = _nextind_str(s, i)
+@propagate_inbounds nextind(s::String, i::Int) = _nextind_str(s, i)
 
 # s should be String or SubString{String}
 @inline function _nextind_str(s, i::Int)
@@ -251,7 +252,7 @@ getindex(s::String, r::UnitRange{<:Integer}) = s[Int(first(r)):Int(last(r))]
     j = nextind(s, j) - 1
     n = j - i + 1
     ss = _string_n(n)
-    unsafe_copyto!(pointer(ss), pointer(s, i), n)
+    GC.@preserve s ss unsafe_copyto!(pointer(ss), pointer(s, i), n)
     return ss
 end
 
@@ -323,7 +324,7 @@ function repeat(c::Char, r::Integer)
     n = 4 - (leading_zeros(u | 0xff) >> 3)
     s = _string_n(n*r)
     p = pointer(s)
-    if n == 1
+    GC.@preserve s if n == 1
         ccall(:memset, Ptr{Cvoid}, (Ptr{UInt8}, Cint, Csize_t), p, u % UInt8, r)
     elseif n == 2
         p16 = reinterpret(Ptr{UInt16}, p)
@@ -340,7 +341,7 @@ function repeat(c::Char, r::Integer)
             unsafe_store!(p, b3, 3i + 3)
         end
     elseif n == 4
-        p32 = reinterpret(Ptr{UInt32}, pointer(s))
+        p32 = reinterpret(Ptr{UInt32}, p)
         for i = 1:r
             unsafe_store!(p32, u, i)
         end
@@ -349,11 +350,11 @@ function repeat(c::Char, r::Integer)
 end
 
 function filter(f, s::String)
-    out = Base.StringVector(sizeof(s))
+    out = StringVector(sizeof(s))
     offset = 1
     for c in s
         if f(c)
-            offset += Base.__unsafe_string!(out, c, offset)
+            offset += __unsafe_string!(out, c, offset)
         end
     end
     resize!(out, offset-1)

base/strings/substring.jl

@@ -51,7 +51,11 @@ convert(::Type{SubString{S}}, s::AbstractString) where {S<:AbstractString} =
     SubString(convert(S, s))
 convert(::Type{T}, s::T) where {T<:SubString} = s
 
-String(s::SubString{String}) = unsafe_string(pointer(s.string, s.offset+1), s.ncodeunits)
+function String(s::SubString{String})
+    parent = s.string
+    copy = GC.@preserve parent unsafe_string(pointer(parent, s.offset+1), s.ncodeunits)
+    return copy
+end
 
 ncodeunits(s::SubString) = s.ncodeunits
 codeunit(s::SubString) = codeunit(s.string)
@@ -151,25 +155,27 @@ end
 string(a::String)            = String(a)
 string(a::SubString{String}) = String(a)
 
-@inline function __unsafe_string!(out, c::Char, offs::Integer)
+@inline function __unsafe_string!(out, c::Char, offs::Integer) # out is a (new) String (or StringVector)
     x = bswap(reinterpret(UInt32, c))
     n = ncodeunits(c)
-    unsafe_store!(pointer(out, offs), x % UInt8)
-    n == 1 && return n
-    x >>= 8
-    unsafe_store!(pointer(out, offs+1), x % UInt8)
-    n == 2 && return n
-    x >>= 8
-    unsafe_store!(pointer(out, offs+2), x % UInt8)
-    n == 3 && return n
-    x >>= 8
-    unsafe_store!(pointer(out, offs+3), x % UInt8)
+    GC.@preserve out begin
+        unsafe_store!(pointer(out, offs), x % UInt8)
+        n == 1 && return n
+        x >>= 8
+        unsafe_store!(pointer(out, offs+1), x % UInt8)
+        n == 2 && return n
+        x >>= 8
+        unsafe_store!(pointer(out, offs+2), x % UInt8)
+        n == 3 && return n
+        x >>= 8
+        unsafe_store!(pointer(out, offs+3), x % UInt8)
+    end
     return n
 end
 
 @inline function __unsafe_string!(out, s::Union{String, SubString{String}}, offs::Integer)
     n = sizeof(s)
-    unsafe_copyto!(pointer(out, offs), pointer(s), n)
+    GC.@preserve s out unsafe_copyto!(pointer(out, offs), pointer(s), n)
     return n
 end
 
@@ -200,7 +206,7 @@ function repeat(s::Union{String, SubString{String}}, r::Integer)
         ccall(:memset, Ptr{Cvoid}, (Ptr{UInt8}, Cint, Csize_t), out, b, r)
     else
         for i = 0:r-1
-            unsafe_copyto!(pointer(out, i*n+1), pointer(s), n)
+            GC.@preserve s out unsafe_copyto!(pointer(out, i*n+1), pointer(s), n)
         end
     end
     return out

base/strings/util.jl

@@ -442,7 +442,7 @@ function replace(str::String, pat_repl::Pair; count::Integer=typemax(Int))
     out = IOBuffer(sizehint=floor(Int, 1.2sizeof(str)))
     while j != 0
         if i == a || i <= k
-            unsafe_write(out, pointer(str, i), UInt(j-i))
+            GC.@preserve str unsafe_write(out, pointer(str, i), UInt(j-i))
             _replace(out, repl, str, r, pattern)
         end
         if k < j

base/twiceprecision.jl

@@ -393,7 +393,7 @@ function (:)(start::T, step::T, stop::T) where T<:Union{Float16,Float32,Float64}
         stop_n, stop_d = rat(stop)
         if start_d != 0 && stop_d != 0 &&
                 T(start_n/start_d) == start && T(stop_n/stop_d) == stop
-            den = lcm(start_d, step_d) # use same denominator for start and step
+            den = lcm_unchecked(start_d, step_d) # use same denominator for start and step
             m = maxintfloat(T, Int)
             if den != 0 && abs(start*den) <= m && abs(step*den) <= m &&  # will round succeed?
                     rem(den, start_d) == 0 && rem(den, step_d) == 0      # check lcm overflow
@@ -429,7 +429,7 @@ function _range(a::T, st::T, ::Nothing, len::Integer) where T<:Union{Float16,Flo
     step_n, step_d = rat(st)
     if start_d != 0 && step_d != 0 &&
             T(start_n/start_d) == a && T(step_n/step_d) == st
-        den = lcm(start_d, step_d)
+        den = lcm_unchecked(start_d, step_d)
         m = maxintfloat(T, Int)
         if abs(den*a) <= m && abs(den*st) <= m &&
                 rem(den, start_d) == 0 && rem(den, step_d) == 0
@@ -513,7 +513,7 @@ function _convertSRL(::Type{StepRangeLen{T,R,S}}, r::AbstractRange{U}) where {T,
     step_n, step_d = rat(s)
     if start_d != 0 && step_d != 0 &&
             U(start_n/start_d) == f && U(step_n/step_d) == s
-        den = lcm(start_d, step_d)
+        den = lcm_unchecked(start_d, step_d)
         m = maxintfloat(T, Int)
         if den != 0 && abs(f*den) <= m && abs(s*den) <= m &&
                 rem(den, start_d) == 0 && rem(den, step_d) == 0
@@ -582,7 +582,7 @@ function _range(start::T, ::Nothing, stop::T, len::Integer) where {T<:IEEEFloat}
     start_n, start_d = rat(start)
     stop_n, stop_d = rat(stop)
     if start_d != 0 && stop_d != 0
-        den = lcm(start_d, stop_d)
+        den = lcm_unchecked(start_d, stop_d)
         m = maxintfloat(T, Int)
         if den != 0 && abs(den*start) <= m && abs(den*stop) <= m
             start_n = round(Int, den*start)
@@ -691,6 +691,9 @@ function rat(x)
     return a, b
 end
 
+# This version of lcm does not check for overflows
+lcm_unchecked(a::T, b::T) where T<:Integer = a * div(b, gcd(a, b))
+
 narrow(::Type{T}) where {T<:AbstractFloat} = Float64
 narrow(::Type{Float64}) = Float32
 narrow(::Type{Float32}) = Float16

contrib/fixup-libgfortran.sh

@@ -3,6 +3,7 @@
 
 # Run as: fixup-libgfortran.sh [--verbose] <$private_libdir>
 FC=${FC:-gfortran}
+PATCHELF=${PATCHELF:-patchelf}
 
 # If we're invoked with "--verbose", create a `debug` function that prints stuff out
 if [ "$1" = "--verbose" ] || [ "$1" = "-v" ]; then
@@ -126,7 +127,7 @@ change_linkage()
         echo " $old_link"
         install_name_tool -change "$old_link" "@rpath/$soname" "$lib_path"
     else # $UNAME is "Linux", we only have two options, see above
-        patchelf --set-rpath \$ORIGIN "$lib_path"
+        ${PATCHELF} --set-rpath \$ORIGIN "$lib_path"
     fi
 }
 

deps/Versions.make

@@ -40,4 +40,4 @@ P7ZIP_BB_REL = 1
 # Specify the version of the Mozilla CA Certificate Store to obtain.
 # The versions of cacert.pem are identified by the date (YYYY-MM-DD) of their changes.
 # See https://curl.haxx.se/docs/caextract.html for more details.
-MOZILLA_CACERT_VERSION := 2019-08-28
+MOZILLA_CACERT_VERSION := 2019-10-16

deps/checksums/Pkg-f71e2c5a119b9c850f9b357fc8c56068f5b51cc0.tar.gz/md5

@@ -0,0 +1 @@
+054eb913fe9bcbe6080c2305980a6c6e

deps/checksums/Pkg-f71e2c5a119b9c850f9b357fc8c56068f5b51cc0.tar.gz/sha512

@@ -0,0 +1 @@
+31c71b702b2831efe08911cab73b78aa2988524a5563720da902d1012e621c45e6bcad7310abc291e934be0d3f481d884af83e3cb4c63e284d8351938faf211f

deps/checksums/cacert-2019-10-16.pem/md5

@@ -0,0 +1 @@
+5805059ab9e4646e4803ce1e007eb8ba

deps/checksums/cacert-2019-10-16.pem/sha512

@@ -0,0 +1 @@
+49778472e46ce3b86b3930f4df5731ac86daf4d8602d418af1c89dc35df5f98c4557aa6c6eb280558c61139ead4b96cbb457a259f72640452f28a2fecd4ccb89